From e467a51250149d731363e33150ff14fdf5d3ad01 Mon Sep 17 00:00:00 2001 From: George Thomas <98996322+george42-ctds@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:40:30 -0700 Subject: [PATCH 01/10] chore/read gen3 license from g3auto secret (#2289) * chore/read gen3 license from g3auto secret --- .../jobs/distribute-licenses-job.yaml | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/kube/services/jobs/distribute-licenses-job.yaml b/kube/services/jobs/distribute-licenses-job.yaml index aef52c75c..1c2ad4284 100644 --- a/kube/services/jobs/distribute-licenses-job.yaml +++ b/kube/services/jobs/distribute-licenses-job.yaml @@ -48,10 +48,11 @@ spec: configMapKeyRef: name: manifest-hatchery key: "user-namespace" - - name: GEN3_LICENSE_SECRET_NAME - value: stata-workspace-gen3-license - - name: GEN3_LICENSE_KEY - value: licenseSecrets + - name: GEN3_STATA_LICENSE + valueFrom: + secretKeyRef: + name: stata-workspace-gen3-license-g3auto + key: "stata_license.txt" command: ["python"] args: - "-c" @@ -100,19 +101,10 @@ spec: used_licenses.sort() print(f"Licenses currently in use: {used_licenses}") - # The license keys should be stored in a kubernetes secret. + # The Gen3 Stata license strings should be stored in a kubernetes secret using g3auto. # The format of the secret is one license string per line. # The license strings are generated with 'stinit' using the information in a license PDF. - # The secret can be generated from a temporary file with a kubectl command, eg - # kubectl create secret generic GEN3_LICENSE_SECRET_NAME --from-file=GEN3_LICENSE_KEY=/path/to/file.lic - - # Get license from kubernetes secret - print("Ready to read secret") - secret_name = os.environ['GEN3_LICENSE_SECRET_NAME'] - secret_key = os.environ['GEN3_LICENSE_KEY'] - license_secrets = os.popen( - f"kubectl get secret {secret_name} --template={{{{.data.{secret_key}}}}} | base64 -d" - ).read() + license_secrets = os.environ['GEN3_STATA_LICENSE'] license_secrets = license_secrets.strip() licenses = license_secrets.split("\n") From bdafe3a998577b4adeb61a9be60110a7f6711d02 Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Sun, 30 Jul 2023 09:43:44 -0700 Subject: [PATCH 02/10] Update jenkins to latest versions (#2297) * Update jenkins to latest versions * remove python-pip * remove duplicate install of python and awscli * fix * fix * fix * fix * build jenkins images for amd64 alone * fix * fix * fix * fix --- .../workflows/image_build_push_jenkins.yaml | 4 ++ .secrets.baseline | 10 ++--- Docker/jenkins/Jenkins-CI-Worker/Dockerfile | 39 +++++++------------ Docker/jenkins/Jenkins-Worker/Dockerfile | 31 +++++++-------- Docker/jenkins/Jenkins/Dockerfile | 29 ++++++-------- Docker/jenkins/Jenkins2/Dockerfile | 26 ++++++------- 6 files changed, 62 insertions(+), 77 deletions(-) diff --git a/.github/workflows/image_build_push_jenkins.yaml b/.github/workflows/image_build_push_jenkins.yaml index ffea50ace..2d85aedf1 100644 --- a/.github/workflows/image_build_push_jenkins.yaml +++ b/.github/workflows/image_build_push_jenkins.yaml @@ -14,6 +14,7 @@ jobs: DOCKERFILE_BUILD_CONTEXT: "./Docker/jenkins/Jenkins" OVERRIDE_REPO_NAME: "jenkins" USE_QUAY_ONLY: true + BUILD_PLATFORMS: "linux/amd64" secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} @@ -27,6 +28,7 @@ jobs: DOCKERFILE_BUILD_CONTEXT: "./Docker/jenkins/Jenkins2" OVERRIDE_REPO_NAME: "jenkins2" USE_QUAY_ONLY: true + BUILD_PLATFORMS: "linux/amd64" secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} @@ -40,6 +42,7 @@ jobs: DOCKERFILE_BUILD_CONTEXT: "./Docker/jenkins/Jenkins-CI-Worker" OVERRIDE_REPO_NAME: "gen3-ci-worker" USE_QUAY_ONLY: true + BUILD_PLATFORMS: "linux/amd64" secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} @@ -53,6 +56,7 @@ jobs: DOCKERFILE_BUILD_CONTEXT: "./Docker/jenkins/Jenkins-Worker" OVERRIDE_REPO_NAME: "gen3-qa-worker" USE_QUAY_ONLY: true + BUILD_PLATFORMS: "linux/amd64" secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} diff --git a/.secrets.baseline b/.secrets.baseline index 621c0a009..791bab52e 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-06-06T18:46:35Z", + "generated_at": "2023-07-26T18:54:08Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -78,7 +78,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 122, + "line_number": 113, "type": "Secret Keyword" } ], @@ -86,7 +86,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 136, + "line_number": 135, "type": "Secret Keyword" } ], @@ -94,7 +94,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 110, + "line_number": 105, "type": "Secret Keyword" } ], @@ -102,7 +102,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 110, + "line_number": 106, "type": "Secret Keyword" } ], diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index afb1fca9f..8c6c78325 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -1,12 +1,9 @@ -FROM jenkins/jnlp-slave:4.13.3-1-jdk11 +FROM jenkins/inbound-agent:jdk11 USER root ENV DEBIAN_FRONTEND=noninteractive -# install python -RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base wget - RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ @@ -16,7 +13,6 @@ RUN set -xe && apt-get update \ libffi-dev \ libssl-dev \ libghc-regex-pcre-dev \ - linux-headers-amd64 \ libcurl4-openssl-dev \ libncurses5-dev \ libncursesw5-dev \ @@ -27,12 +23,12 @@ RUN set -xe && apt-get update \ libbz2-dev \ libexpat1-dev \ liblzma-dev \ - python-virtualenv \ lua5.3 \ r-base \ software-properties-common \ sudo \ tk-dev \ + wget \ zlib1g-dev \ zsh \ ca-certificates-java \ @@ -58,30 +54,25 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \ # # install docker tools: -# * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1 -# * https://docs.docker.com/compose/install/#install-compose # -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/debian \ - $(lsb_release -cs) \ - stable" \ - && apt-get update \ - && apt-get install -y docker-ce \ - && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ - && chmod a+rx /usr/local/bin/docker-compose +RUN sudo install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && sudo chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update && apt-get install -y docker-ce # install nodejs RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get update && apt-get install -y nodejs -# add psql: https://www.postgresql.org/download/linux/debian/ -RUN DISTRO="$(lsb_release -c -s)" \ - && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRO}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ - && apt-get update \ - && apt-get install -y postgresql-client-13 libpq-dev \ - && rm -rf /var/lib/apt/lists/* +# Install postgres 13 client +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc| gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg && \ + echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ + apt-get update && \ + apt-get install -y postgresql-client-13 # Copy sh script responsible for installing Python COPY install-python3.8.sh /root/tmp/install-python3.8.sh diff --git a/Docker/jenkins/Jenkins-Worker/Dockerfile b/Docker/jenkins/Jenkins-Worker/Dockerfile index 7b1d460cc..61216733a 100644 --- a/Docker/jenkins/Jenkins-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-Worker/Dockerfile @@ -1,16 +1,9 @@ -FROM jenkins/jnlp-slave:4.13.3-1-jdk11 +FROM jenkins/inbound-agent:jdk11 USER root ENV DEBIAN_FRONTEND=noninteractive -# install python and pip and aws cli -RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip build-essential libgit2-dev zip unzip less vim gettext-base wget -RUN set -xe && python -m pip install awscli --upgrade && python -m pip install pytest --upgrade && python -m pip install PyYAML --upgrade && python -m pip install lxml --upgrade -RUN set -xe && python3 -m pip install pytest --upgrade && python3 -m pip install PyYAML --upgrade -RUN set -xe && python -m pip install yq --upgrade && python3 -m pip install yq --upgrade -RUN set -xe && python3 -m pip install pandas --upgrade - RUN apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ @@ -35,6 +28,7 @@ RUN apt-get update \ lua5.3 \ software-properties-common \ sudo \ + wget \ && ln -s /usr/bin/lua5.3 /usr/local/bin/lua # install Ruby. @@ -45,11 +39,17 @@ RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sou && apt-get update \ && apt-get -t=buster-backports -y install git=1:2.30.* -# install k6 to run load tests -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 \ - && echo "deb https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list \ - && apt-get update \ - && apt-get install k6 +# +# install docker tools: +# +RUN sudo install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && sudo chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update && apt-get install -y docker-ce # install xk6-browser RUN cd /opt && wget --quiet https://github.com/grafana/xk6-browser/releases/download/v0.3.0/xk6-browser-v0.3.0-linux-amd64.tar.gz \ @@ -71,15 +71,13 @@ RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && c # # install docker tools: -# * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1 -# * https://docs.docker.com/compose/install/#install-compose # RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && /usr/bin/add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian \ $(lsb_release -c -s) \ stable" \ && apt-get update \ - && apt-get install -y docker-ce \ + && apt-get install -y docker-ce-cli \ && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ && chmod a+rx /usr/local/bin/docker-compose @@ -118,6 +116,7 @@ RUN chmod +x /root/tmp/install-python3.9.sh; sync && \ bash /root/tmp/install-python3.9.sh && \ rm -rf /root/tmp/install-python3.9.sh && \ unlink /usr/bin/python3 && \ + ln -s /usr/local/bin/python3.9 /usr/bin/python && \ ln -s /usr/local/bin/python3.9 /usr/bin/python3 RUN env diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index a872ee1dd..e6cf065db 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -1,12 +1,9 @@ -FROM jenkins/jenkins:2.375 +FROM jenkins/jenkins:2.415-jdk11 USER root ENV DEBIAN_FRONTEND=noninteractive -# install python -RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base wget - RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ @@ -30,6 +27,7 @@ RUN set -xe && apt-get update \ software-properties-common \ sudo \ tk-dev \ + wget \ zlib1g-dev \ zsh \ && ln -s /usr/bin/lua5.3 /usr/local/bin/lua @@ -45,18 +43,15 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \ # # install docker tools: -# * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1 -# * https://docs.docker.com/compose/install/#install-compose # -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/debian \ - $(lsb_release -cs) \ - stable" \ - && apt-get update \ - && apt-get install -y docker-ce \ - && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ - && chmod a+rx /usr/local/bin/docker-compose +RUN sudo install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && sudo chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update && apt-get install -y docker-ce # install nodejs RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - @@ -81,8 +76,8 @@ RUN chmod +x /root/tmp/install-python3.8.sh; sync && \ ln -s /Python-3.8.0/python /usr/bin/python3 # Fix shebang for lsb_release -RUN sed -i 's/python3/python3.5/' /usr/bin/lsb_release && \ - sed -i 's/python3/python3.5/' /usr/bin/add-apt-repository +RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \ + sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository # install aws cli, poetry, pytest, etc. RUN set -xe && python3 -m pip install --upgrade pip && python3 -m pip install awscli --upgrade && python3 -m pip install pytest --upgrade && python3 -m pip install poetry && python3 -m pip install PyYAML --upgrade && python3 -m pip install lxml --upgrade && python3 -m pip install yq --upgrade diff --git a/Docker/jenkins/Jenkins2/Dockerfile b/Docker/jenkins/Jenkins2/Dockerfile index 59cb5672e..45f8fb373 100644 --- a/Docker/jenkins/Jenkins2/Dockerfile +++ b/Docker/jenkins/Jenkins2/Dockerfile @@ -1,12 +1,9 @@ -FROM jenkins/jenkins:2.375 +FROM jenkins/jenkins:2.415-jdk11 USER root ENV DEBIAN_FRONTEND=noninteractive -# install python -RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base wget - RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ @@ -30,6 +27,7 @@ RUN set -xe && apt-get update \ software-properties-common \ sudo \ tk-dev \ + wget \ zlib1g-dev \ zsh \ && ln -s /usr/bin/lua5.3 /usr/local/bin/lua @@ -45,18 +43,16 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \ # # install docker tools: -# * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1 -# * https://docs.docker.com/compose/install/#install-compose # -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/debian \ - $(lsb_release -cs) \ - stable" \ - && apt-get update \ - && apt-get install -y docker-ce \ - && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ - && chmod a+rx /usr/local/bin/docker-compose +RUN sudo install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && sudo chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update && apt-get install -y docker-ce + # install nodejs RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - From f128dc892381cc403e5d40cb564259790ed984ad Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:33:19 -0600 Subject: [PATCH 03/10] removing squid cache to see if that resolves self-signed certificate errors we have been seeing with our Java applications. (#2298) --- flavors/squid_auto/startup_configs/squid.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/flavors/squid_auto/startup_configs/squid.conf b/flavors/squid_auto/startup_configs/squid.conf index 653026200..b1e44810a 100644 --- a/flavors/squid_auto/startup_configs/squid.conf +++ b/flavors/squid_auto/startup_configs/squid.conf @@ -56,7 +56,6 @@ http_access deny all persistent_request_timeout 5 seconds -cache_dir ufs /var/cache/squid 100 16 256 pid_filename /var/run/squid/squid.pid # vi:syntax=squid.conf From 31a5277d29f82c0974788aefaa4ba1520ea263c6 Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Mon, 31 Jul 2023 10:14:22 -0700 Subject: [PATCH 04/10] Add missing jenkins dependencies (#2299) --- .secrets.baseline | 10 +++++----- Docker/jenkins/Jenkins-CI-Worker/Dockerfile | 2 ++ Docker/jenkins/Jenkins-Worker/Dockerfile | 2 ++ Docker/jenkins/Jenkins/Dockerfile | 2 ++ Docker/jenkins/Jenkins2/Dockerfile | 2 ++ 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 791bab52e..8e671afaa 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-07-26T18:54:08Z", + "generated_at": "2023-07-31T16:54:24Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -78,7 +78,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 113, + "line_number": 115, "type": "Secret Keyword" } ], @@ -86,7 +86,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 135, + "line_number": 137, "type": "Secret Keyword" } ], @@ -94,7 +94,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 105, + "line_number": 107, "type": "Secret Keyword" } ], @@ -102,7 +102,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 106, + "line_number": 108, "type": "Secret Keyword" } ], diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index 8c6c78325..671cd2e02 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -4,6 +4,8 @@ USER root ENV DEBIAN_FRONTEND=noninteractive +RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-essential zip unzip jq less vim gettext-base + RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ diff --git a/Docker/jenkins/Jenkins-Worker/Dockerfile b/Docker/jenkins/Jenkins-Worker/Dockerfile index 61216733a..088186b04 100644 --- a/Docker/jenkins/Jenkins-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-Worker/Dockerfile @@ -4,6 +4,8 @@ USER root ENV DEBIAN_FRONTEND=noninteractive +RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-essential zip unzip jq less vim gettext-base + RUN apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index e6cf065db..ae39ac574 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -4,6 +4,8 @@ USER root ENV DEBIAN_FRONTEND=noninteractive +RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-essential zip unzip jq less vim gettext-base + RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ diff --git a/Docker/jenkins/Jenkins2/Dockerfile b/Docker/jenkins/Jenkins2/Dockerfile index 45f8fb373..9976a07c2 100644 --- a/Docker/jenkins/Jenkins2/Dockerfile +++ b/Docker/jenkins/Jenkins2/Dockerfile @@ -4,6 +4,8 @@ USER root ENV DEBIAN_FRONTEND=noninteractive +RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-essential zip unzip jq less vim gettext-base + RUN set -xe && apt-get update \ && apt-get install -y lsb-release \ apt-transport-https \ From 5c0866a3a70f97d583c529e2aa656abb21d3adf6 Mon Sep 17 00:00:00 2001 From: burtonk <117617405+k-burt-uch@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:15:26 -0500 Subject: [PATCH 05/10] Update squid_authorized_keys_user (#2291) Adding Kyle Burton's public key. --- files/authorized_keys/squid_authorized_keys_user | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/files/authorized_keys/squid_authorized_keys_user b/files/authorized_keys/squid_authorized_keys_user index 46b43a030..4b35fecd9 100644 --- a/files/authorized_keys/squid_authorized_keys_user +++ b/files/authorized_keys/squid_authorized_keys_user @@ -18,4 +18,5 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKJR5N5VIU9qdSfCtlskzuQ7A5kNn8YPeXsoKq0HhY ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3vyd6a7tsANi149ylPQYS8Gsp/SxJyhdK/j6arv77KbM0EIzzUiclFLnMKcqUQ263FrPyx3a3UP80R77ayCnwcEHrxlJrYfyFUva8vtmI9mu8VE7oXvuR/jcOyXM9NosxyYacL/p6W5X4r8tqo/gJFjmls1YRfu3JPlTgTT0VzGJu+B6rLEsw53c37VVzSaCtu/jBOjyxI1/UaNg1cd+hcfoQxJ9zSDqqE7ZUNOc3zHP+1AGYCQ/CJsNrDl2OkppIdC9He5jgjLhyD7yvyarI+oF05oHknol/K1hXK+yxIkF2Ou5krfjw7TMBvD+JbQVb35vL9acXFF20+lHLRLbobPU/6ZZTup3q7IRm5OWaL2CJtYZbJvicKW0Ep+vTzaiQjK71L6UxcIvnzvbP9Dnatv1GBMMDaQxAa4Lood8NG2ty1yfLN972akGqBlwJASXMRd/ogzxv2KSH9w6HHYoc2WpDhUtNHmjwX1FSLYPW3qx5ICMW6j9gR2u1tG4Ohzp1CmYVElnRHbnBrTkLde65Vqedk2tQy8fcopH59ZASIuR4GbhCb2SiNkr1PHEvfhLMzg/UCSnnhX9vUNhkPjJRy/bdL3pOt/77lpIQUqQBArOiZmfG8OD0q4+3Nr+c9v5bSSvynjirlKk+wb8sKyOoSAXdFeovL/A0BUKUjCtsXQ== dev@test.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQChK/8JjpUeWcF/1Ea2M4mSbLz1tOfpq74xD2USxE54kx7VoN1G7ylV76yqSIeRq1e7PPBEg5ZD1aXUJnlI32RwLJ5kaHnoB82Ta+Fv1B/vVoHCObcALfiHPpwPf1kM2liWEB0EhYcz1OUv3YQriPqjiRoWfnbw60GIyzhpWZhKRq0zlISOaTYdV9kafX+N7M6/gSU0632TgUwwsStYrffEleyrC/Lh+4UaESozWoPFiZLl2eMCKfZNFBB99HTFifImW2yC6Ag1QhCd1i3NpfiYuaSDH7WR3slPRSd8DiUAwGC2DkIuWPp3bhaAv2V4mtLIBAaTZsINIACB2+w7yf9yvCGtdobCmp4AA7ik9rEkRLk/Jff0YBHd6Z4qyIuRht3ZeWXIYSK1zOlPfs4lPUgvbjlPgMVFV2CrvOTnS+YZdW+8AklwRC3HDPD8wv3H/eGxl3K0vHWTBbTb774nVNfRDw81wcezCXFNUn4p2he7fgKcxs/rnMsYUcY8JJNR7Iz+NNIGUCom6HFwCMQdangFMHUW5TxxrlJcwVRaAns1M6g3ilYO+uvN/XsgCpZWYWnv5rBk8qz6dBM7gpc8tSr6Hvr7/vlghF3jpL+mQiW+7vUL+UZrUFNyoacUcQ+NuxKacHtHQKuRDyWofp+CB2b2a744F3mpkxx74HIkiZ72mQ== dev@test.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDTX+pQvGrQVXmHGDhBP+632tgbb1j+BQWkrsUkDJGzwFiGs4dgqDs2eC+aDVq2LFz4xj0SgussFAKciB45OgmSZKX5yUE3Oo/lqov0Bb5f85iBHGv/X/JiuIYaq8GJklVyyo1sfKLUK1SOal6bE1WofezyTyDsdrHjIU50quzW7nB1CmL6rekIv/+df/seut4b3De1d2uX5WGGtcvQ5yTSgBW5aabMAJ2V9WlP/6Dw040Kq0MyKV01cIJ1HAjFhP58gbf3Eytz3AqqJVT6u0QroxhesCgKTyGcAyYy3airI/N0FHdC5oABVEJ6dKyy1rYvOchuxYeVMVVWn0vS7mZ+vP7dqaDmgEUU2qmTPBQZV2xBWCdpfyUYYARW2JzlEaySbmA+yoxFBsquunVbIgUGNEUbxefsFdM3k5pS6I1uuEM0ATYH5iNz84nKKCcksGlib0i/pEtra6N/mFF7yjHYBRb/E/VCZig0gKezDJWu/DO0emJA+kdQpqp48U+qFrSWkuiO0dCQYl3VCVo8vedgMGPjr8MbUjU7o8W1+DYyjFM8HYMknRNdVAqAoK+cedw9mAWVGpKFrl61caGTFck0634nAVFUmfGTh9XRaZeFdDnivxnqP837gcsdKnEGYnkrxWap97XeXzK0P0Svy1zBfUQyzU5vrHfHt2H7ILDMw== prodv1-usersync-sftp -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDaO/doqHANcTZFEqZOoy9kKgbxu0d/cS1nEINlFcoQ/jnCG7huznWnWiYgnlkS6/Op9VrDp6qG/UBDye2mTvAh2FHPsOzSGvgml3dPYB5fy6G/xoXd7NJnIxttwFUvk4GuLZ40s24WCcXoFGJ2vaSAVYr0q6lmqOqk6jp1/lNj4+QFD4mcH2//jTscSFNseRII2NECu+PnnWAuYFOIHH1IODOvInEivUvN6VBX410D7iD7cEdhgiYitFZH6Cp6ubWG7OUKdZYv0067eO6HDDzl7y+BBUf3DF6Lr8gqtGXVqmAB9UqeBJ8pP3pNWKbgAa8sHvS8JxElCIc+4EM5dTI2OrDYKiuCTPZEC14WEFZLKqH7tjQFuZe0jfVRtoFNmKWClCgkJDWpyIkdR+qHcnOwlYkUVN3B02WVu4kTfox2ZUz65tLspJNAxAjYVrI7+c6LTQHSJwMcAMYcehR3vuqAfKE7xM6ReNxRQXsWaasdJgT2IJKj7vHu/G9GVycjiheg3zakJ9rr+63I68XlHNnTtfjIl/jgIHgcU18ggbwkwjL3xk39YttutlAaNAGUYCsopn/HdK8A86KvTCwHGEKtubgEHmv1oRAOooVaNes1oko2y9Saaqee52bsvwfeTLgxXB43d9GOWLoyBlgprDiufssFHoiJKQlgrqEwtg+vYQ== giangbui0816@gmail.com \ No newline at end of file +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDaO/doqHANcTZFEqZOoy9kKgbxu0d/cS1nEINlFcoQ/jnCG7huznWnWiYgnlkS6/Op9VrDp6qG/UBDye2mTvAh2FHPsOzSGvgml3dPYB5fy6G/xoXd7NJnIxttwFUvk4GuLZ40s24WCcXoFGJ2vaSAVYr0q6lmqOqk6jp1/lNj4+QFD4mcH2//jTscSFNseRII2NECu+PnnWAuYFOIHH1IODOvInEivUvN6VBX410D7iD7cEdhgiYitFZH6Cp6ubWG7OUKdZYv0067eO6HDDzl7y+BBUf3DF6Lr8gqtGXVqmAB9UqeBJ8pP3pNWKbgAa8sHvS8JxElCIc+4EM5dTI2OrDYKiuCTPZEC14WEFZLKqH7tjQFuZe0jfVRtoFNmKWClCgkJDWpyIkdR+qHcnOwlYkUVN3B02WVu4kTfox2ZUz65tLspJNAxAjYVrI7+c6LTQHSJwMcAMYcehR3vuqAfKE7xM6ReNxRQXsWaasdJgT2IJKj7vHu/G9GVycjiheg3zakJ9rr+63I68XlHNnTtfjIl/jgIHgcU18ggbwkwjL3xk39YttutlAaNAGUYCsopn/HdK8A86KvTCwHGEKtubgEHmv1oRAOooVaNes1oko2y9Saaqee52bsvwfeTLgxXB43d9GOWLoyBlgprDiufssFHoiJKQlgrqEwtg+vYQ== giangbui0816@gmail.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDTpJ2l8nfOvhJ4Y3pjadFU69nfJBRuE0BaHE22LK9qflFWdhGW+T/x8Qy9406DFXCh6KED+q9lC+N4nR92AfgFNaBmkXZkzWLoXcqO1IWRexHwTqAUcrtLjpx5wNdCr3+vv9hWhXtvYg8ewnrZc+WxYde4EUmXbhzPXbg0SkBXTr6bpYhs6inyttfBeJNxbeydrW7cmhFiAdOkm03o3AXdH86PNlWVfVHy8OHHzf4fbvlJlOx7OeB+wOyQUr3DW+IWBLQFJk4uyagn/ECV9OIQpxoJFTQjcSrJ6v/GqlY5PImM6YxL8NlZu46CDIxagaZkum+iJ8dtPYr6tJuLiP5Ny0Gsl1X5DoKlstgyqqPNYTnZVS4GSS5Hyxm6HmodZ78OR5+vAoyWKZ3unXU5Dbkz0Qxq9VtrGo2xd0M+dDi/7YazRpLL0tc39w48Wl7KD3jFzoesZp1JHeEGLdGXlGCw8AM1FT0WDf28ShTRds6uWPGvMtM3XkVDPMLFwroKv1RCErmqLYod4HOMuwlmdRvtDGYb3NYsliOnHPiT9nhu2J6KmT1jj8uFOLyTaJCArtBqIsXscP3R4o0wBlQl3FniMdiK7ESkv8DUaOr1Co+/3wX9n/p/BW5bxuq1R9HpNyKsrALyNJUkquVT+5aPcNKXvmAeHAw/D0TYzy6ZKBpnDw== kyuleburton@Kyules-MacBook-Pro.local From c89994e4d7dee99126ef14cdf6347fd303130ee7 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Mon, 31 Jul 2023 14:24:04 -0400 Subject: [PATCH 06/10] Feat/update argo events config (#2300) * Matching workflow node tags to our current setup * Fixed a typo in the previous commit --- kube/services/argo-events/workflows/configmap.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index d9ad3d413..9846ba8b6 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -54,8 +54,9 @@ data: Environment: vhdcperf Name: eks-vhdcperf-workflow-karpenter karpenter.sh/discovery: vhdcperf - workflow-name: $WORKFLOW_NAME - gen3-username: $GEN3_USERNAME + workflowname: $WORKFLOW_NAME + gen3username: $GEN3_USERNAME + gen3service: argo-workflows metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled From 02ab503ee6fcaae30f20c67a7b7d27c6e9aa22e3 Mon Sep 17 00:00:00 2001 From: Michael Lukowski Date: Mon, 31 Jul 2023 16:20:09 -0500 Subject: [PATCH 07/10] HP-1083 Feat/cedar mds update (#2268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updating cedar ingest for new MDS format for HEAL * fix for upper level metadata * change where appl_id comes from in cedar ingestion * testing alt appl_id * fix appl id again * update appl_id check * add debug statement for cedar ingestion * remove debug statements and finalize * updateing cedar records in ingestion job * test (#2281) * test * update * test * test * update * test * fix * fix * test * fix * update * update * final * update * test * revert test * fix * test * debug * revert debug * fix: dups * Created a setup script to install argo events, as well as any resourc… (#2287) * Created a setup script to install argo events, as well as any resources used by it. For now, all the resources are for Argo Workflows * Moving from using the public eventbus demo to a local copy, so that updates to the documentation don't break our setup * back to master --------- Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Co-authored-by: Mingfei Shao Co-authored-by: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> --- .../healdata/heal-cedar-data-ingest.py | 106 +++++++++++++----- 1 file changed, 77 insertions(+), 29 deletions(-) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 1235c6f58..c4d68199a 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -1,15 +1,17 @@ import argparse +import json import sys import requests import pydash +from uuid import UUID # Defines how a field in metadata is going to be mapped into a key in filters FILTER_FIELD_MAPPINGS = { - "Study Type.study_stage": "Study Type", - "Data.data_type": "Data Type", - "Study Type.study_subject_type": "Subject Type", - "Human Subject Applicability.gender_applicability": "Gender", - "Human Subject Applicability.age_applicability": "Age" + "study_metadata.study_type.study_stage": "Study Type", + "study_metadata.data.data_type": "Data Type", + "study_metadata.study_type.study_subject_type": "Subject Type", + "study_metadata.human_subject_applicability.gender_applicability": "Gender", + "study_metadata.human_subject_applicability.age_applicability": "Age" } # Defines how to handle special cases for values in filters @@ -31,9 +33,30 @@ # Defines field that we don't want to include in the filters OMITTED_VALUES_MAPPING = { - "Human Subject Applicability.gender_applicability": "Not applicable" + "study_metadata.human_subject_applicability.gender_applicability": "Not applicable" } +def is_valid_uuid(uuid_to_test, version=4): + """ + Check if uuid_to_test is a valid UUID. + + Parameters + ---------- + uuid_to_test : str + version : {1, 2, 3, 4} + + Returns + ------- + `True` if uuid_to_test is a valid UUID, otherwise `False`. + + """ + + try: + uuid_obj = UUID(uuid_to_test, version=version) + except ValueError: + return False + return str(uuid_obj) == uuid_to_test + def update_filter_metadata(metadata_to_update): filter_metadata = [] for metadata_field_key, filter_field_key in FILTER_FIELD_MAPPINGS.items(): @@ -82,9 +105,13 @@ def update_filter_metadata(metadata_to_update): limit = 10 offset = 0 -# initalize this to be bigger than our inital call so we can go through while loop +# initialize this to be bigger than our initial call so we can go through while loop total = 100 +if not is_valid_uuid(dir_id): + print("Directory ID is not in UUID format!") + sys.exit(1) + while((limit + offset <= total)): # Get the metadata from cedar to register print("Querying CEDAR...") @@ -101,60 +128,81 @@ def update_filter_metadata(metadata_to_update): returned_records = len(metadata_return["metadata"]["records"]) print(f"Successfully got {returned_records} record(s) from CEDAR directory") for cedar_record in metadata_return["metadata"]["records"]: - if "appl_id" not in cedar_record: + # get the appl id from cedar for querying in our MDS + cedar_appl_id = pydash.get(cedar_record, "metadata_location.nih_application_id") + if cedar_appl_id is None: print("This record doesn't have appl_id, skipping...") continue - # get the appl id from cedar for querying in our MDS - cedar_appl_id = str(cedar_record["appl_id"]) - # Get the metadata record for the nih_application_id - mds = requests.get(f"http://revproxy-service/mds/metadata?gen3_discovery.appl_id={cedar_appl_id}&data=true") + mds = requests.get(f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true") if mds.status_code == 200: mds_res = mds.json() # the query result key is the record of the metadata. If it doesn't return anything then our query failed. if len(list(mds_res.keys())) == 0 or len(list(mds_res.keys())) > 1: - print("Query returned nothing for ", cedar_appl_id, "appl id") + print("Query returned nothing for", cedar_appl_id, "appl id") continue # get the key for our mds record - cedar_record_id = list(mds_res.keys())[0] + mds_record_guid = list(mds_res.keys())[0] - mds_res = mds_res[cedar_record_id] - mds_cedar_register_data_body = {} + mds_res = mds_res[mds_record_guid] + mds_cedar_register_data_body = {**mds_res} mds_discovery_data_body = {} + mds_clinical_trials = {} if mds_res["_guid_type"] == "discovery_metadata": print("Metadata is already registered. Updating MDS record") elif mds_res["_guid_type"] == "unregistered_discovery_metadata": - print("Metadata is has not been registered. Registering it in MDS record") - continue + print("Metadata has not been registered. Registering it in MDS record") + + if "clinicaltrials_gov" in cedar_record: + mds_clinical_trials = cedar_record["clinicaltrials_gov"] + del cedar_record["clinicaltrials_gov"] + + # some special handing for this field, because its parent will be deleted before we merging the CEDAR and MDS SLMD to avoid duplicated values + cedar_record_other_study_websites = cedar_record.get("metadata_location", {}).get("other_study_websites", []) + del cedar_record["metadata_location"] + + mds_res["gen3_discovery"]["study_metadata"].update(cedar_record) + mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites + + # merge data from cedar that is not study level metadata into a level higher + deleted_keys = [] + for key, value in mds_res["gen3_discovery"]["study_metadata"].items(): + if not isinstance(value, dict): + mds_res["gen3_discovery"][key] = value + deleted_keys.append(key) + for key in deleted_keys: + del mds_res["gen3_discovery"]["study_metadata"][key] + + mds_discovery_data_body = update_filter_metadata(mds_res["gen3_discovery"]) - pydash.merge(mds_discovery_data_body, mds_res["gen3_discovery"], cedar_record) - mds_discovery_data_body = update_filter_metadata(mds_discovery_data_body) mds_cedar_register_data_body["gen3_discovery"] = mds_discovery_data_body + if mds_clinical_trials: + mds_cedar_register_data_body["clinicaltrials_gov"] = {**mds_cedar_register_data_body.get("clinicaltrials_gov", {}), **mds_clinical_trials} + mds_cedar_register_data_body["_guid_type"] = "discovery_metadata" - print("Metadata is now being registered.") - mds_put = requests.put(f"http://revproxy-service/mds/metadata/{cedar_record_id}", + print(f"Metadata {mds_record_guid} is now being registered.") + mds_put = requests.put(f"http://revproxy-service/mds/metadata/{mds_record_guid}", headers=token_header, json = mds_cedar_register_data_body ) if mds_put.status_code == 200: - print(f"Successfully registered: {cedar_record_id}") + print(f"Successfully registered: {mds_record_guid}") else: - print(f"Failed to register: {cedar_record_id}. Might not be MDS admin") + print(f"Failed to register: {mds_record_guid}. Might not be MDS admin") print(f"Status from MDS: {mds_put.status_code}") else: print(f"Failed to get information from MDS: {mds.status_code}") + + else: + print(f"Failed to get information from CEDAR wrapper service: {cedar.status_code}") if offset + limit == total: break offset = offset + limit if (offset + limit) > total: - limit = (offset + limit) - total - - -else: - print(f"Failed to get information from CEDAR wrapper service: {cedar.status_code}") + limit = total - offset From c4d01f0d31235b0d37d7d039b50607270631e522 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Thu, 3 Aug 2023 12:34:06 -0500 Subject: [PATCH 08/10] Changes to OHDSI Atlas deployment to use upstream Docker images (#2301) * feat(ohdsi): init container for a config-local.js file * feat(ohdsi): different mount-point --- kube/services/ohdsi-atlas/ohdsi-atlas-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/ohdsi-atlas/ohdsi-atlas-deploy.yaml b/kube/services/ohdsi-atlas/ohdsi-atlas-deploy.yaml index bf128920e..62265503e 100644 --- a/kube/services/ohdsi-atlas/ohdsi-atlas-deploy.yaml +++ b/kube/services/ohdsi-atlas/ohdsi-atlas-deploy.yaml @@ -72,7 +72,7 @@ spec: volumeMounts: - name: ohdsi-atlas-config-local readOnly: true - mountPath: /usr/share/nginx/html/atlas/js/config-local.js + mountPath: /etc/atlas/config-local.js subPath: config-local.js imagePullPolicy: Always resources: @@ -80,4 +80,4 @@ spec: cpu: 100m memory: 100Mi limits: - memory: 500Mi + memory: 500Mi From 92f8f3e207c062256c698e30374be629ee4a614a Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Mon, 7 Aug 2023 12:08:18 -0400 Subject: [PATCH 09/10] Covering a few extra cases for preventing clickjacking attacks. (#2309) --- .../gen3ff-as-root/frontend-framework-service.conf | 4 ++++ .../gen3.nginx.conf/gen3ff-as-root/portal-service.conf | 3 +++ .../portal-as-root/frontend-framework-service.conf | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/frontend-framework-service.conf b/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/frontend-framework-service.conf index ac2cb75f6..37e7623de 100644 --- a/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/frontend-framework-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/frontend-framework-service.conf @@ -2,6 +2,10 @@ if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; } + + # added to avoid click-jacking attacks + add_header X-Frame-Options "SAMEORIGIN"; + set $proxy_service "frontend-framework"; set $upstream http://frontend-framework-service.$namespace.svc.cluster.local; proxy_pass $upstream; diff --git a/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/portal-service.conf b/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/portal-service.conf index 58f0851d6..75d69c185 100644 --- a/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/portal-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/gen3ff-as-root/portal-service.conf @@ -21,5 +21,8 @@ rewrite ^/(.*)$ /dashboard/Public/maintenance-page/index.html redirect; } + # added to avoid click-jacking attacks + add_header X-Frame-Options "SAMEORIGIN"; + proxy_pass $upstream; } diff --git a/kube/services/revproxy/gen3.nginx.conf/portal-as-root/frontend-framework-service.conf b/kube/services/revproxy/gen3.nginx.conf/portal-as-root/frontend-framework-service.conf index dbb24e4b2..f3686d1a6 100644 --- a/kube/services/revproxy/gen3.nginx.conf/portal-as-root/frontend-framework-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/portal-as-root/frontend-framework-service.conf @@ -6,6 +6,10 @@ if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; } + + # added to avoid click-jacking attacks + add_header X-Frame-Options "SAMEORIGIN"; + set $proxy_service "frontend-framework"; # frontend framework service expects the /ff/ prefix, so no path rewrite set $upstream http://frontend-framework-service.$namespace.svc.cluster.local; From 311c3aaa63bf58c0e516a233387821fbbe6fd688 Mon Sep 17 00:00:00 2001 From: stevekrasinsky Date: Mon, 7 Aug 2023 15:24:10 -0400 Subject: [PATCH 10/10] Removed deprecated acl settings for data-bucket-with-versioning. --- tf_files/aws/modules/data-bucket-with-versioning/s3.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/tf_files/aws/modules/data-bucket-with-versioning/s3.tf b/tf_files/aws/modules/data-bucket-with-versioning/s3.tf index c69c01471..0dcac8c98 100644 --- a/tf_files/aws/modules/data-bucket-with-versioning/s3.tf +++ b/tf_files/aws/modules/data-bucket-with-versioning/s3.tf @@ -2,7 +2,6 @@ #------------- LOGGING resource "aws_s3_bucket" "log_bucket" { bucket = "${var.vpc_name}-data-bucket-with-versioning-log" - acl = "log-delivery-write" tags = { Purpose = "s3 bucket log bucket" } @@ -10,7 +9,6 @@ resource "aws_s3_bucket" "log_bucket" { resource "aws_s3_bucket" "data_bucket" { bucket = "${var.vpc_name}-data-bucket-with-versioning" - acl = "private" versioning { enabled = true }