Merge branch 'master' into GPE-1081

.github/workflows/image_build_push.yaml

@@ -1,10 +1,10 @@
-name: Build Python Base Images and Push to Quay and ECR
+name: Build Python Base Images
 
 on: push
 
 jobs:
   python_3-9:
-    name: Python 3.9 Build and Push
+    name: Python 3.9
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"
@@ -17,7 +17,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   python_3-10:
-    name: Python 3.10 Build and Push
+    name: Python 3.10
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
@@ -30,7 +30,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   awshelper:
-    name: AwsHelper Build and Push
+    name: AwsHelper
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"

.github/workflows/image_build_push_jenkins.yaml

@@ -1,13 +1,14 @@
-name: Build Jenkins images and push to Quay
+name: Build Jenkins images
 
 on: 
   push:
     paths:
+      - .github/workflows/image_build_push_jenkins.yaml
       - Docker/jenkins/**
 
 jobs:
   jenkins:
-    name: Jenkins Build and Push
+    name: Jenkins
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins/Dockerfile"
@@ -21,7 +22,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}    
   jenkins2:
-    name: Jenkins2 Build and Push
+    name: Jenkins2
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins2/Dockerfile"
@@ -35,7 +36,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   jenkins-ci-worker:
-    name: Jenkins-CI-Worker Build and Push
+    name: Jenkins-CI-Worker
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-CI-Worker/Dockerfile"
@@ -49,7 +50,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   jenkins-qa-worker:
-    name: Jenkins-QA-Worker Build and Push
+    name: Jenkins-QA-Worker
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-Worker/Dockerfile"

.github/workflows/image_build_push_squid.yaml

@@ -1,13 +1,14 @@
-name: Build Squid images and push to Quay
+name: Build Squid images
 
 on: 
   push:
     paths:
+      - .github/workflows/image_build_push_squid.yaml
       - Docker/squid/**
 
 jobs:
   squid:
-    name: Squid Build and Push
+    name: Squid image
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/squid/Dockerfile"

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-09-18T18:49:22Z",
+  "generated_at": "2023-10-26T21:32:44Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -79,7 +79,7 @@
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 122,
+        "line_number": 121,
         "type": "Secret Keyword"
       }
     ],
@@ -342,7 +342,7 @@
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 217,
+        "line_number": 191,
         "type": "Secret Keyword"
       }
     ],

Docker/jenkins/Jenkins-CI-Worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM jenkins/inbound-agent:jdk11
+FROM jenkins/inbound-agent:jdk21
 
 USER root
 
@@ -34,11 +34,10 @@ RUN set -xe && apt-get update \
      zlib1g-dev \
      zsh \
      ca-certificates-java \
-     openjdk-11-jre-headless \
   && ln -s /usr/bin/lua5.3 /usr/local/bin/lua
 
 # Use jdk11
-ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64"
+ENV JAVA_HOME="/opt/java/openjdk"
 ENV PATH="$JAVA_HOME/bin:$PATH"
 
 COPY ./certfix.sh /certfix.sh
@@ -75,7 +74,7 @@ RUN sudo install -m 0755 -d /etc/apt/keyrings \
 
 # install nodejs
 RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get update && apt-get install -y nodejs
+RUN apt-get update && apt-get install -y nodejs npm
 
 # Install postgres 13 client
 RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc| gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg && \
@@ -98,7 +97,7 @@ RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \
     sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3.8 -m pip install --upgrade pip && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade
+RUN set -xe && python3.8 -m pip install --upgrade pip setuptools && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade
 
 # install terraform
 RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \
@@ -117,6 +116,9 @@ RUN curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-ke
     && apt-get -y update \
     && apt-get -y install google-chrome-stable
 
+# data-simulator needs "/usr/share/dict/words" to generate data that isn't random strings
+RUN apt-get install --reinstall wamerican
+
 # update /etc/sudoers
 RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \
   && /bin/echo -e "\n%sudo    ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \

Docker/jenkins/Jenkins-Worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM jenkins/inbound-agent:jdk11
+FROM jenkins/inbound-agent:jdk21
 
 USER root
 
@@ -8,6 +8,7 @@ RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-ess
 
 RUN apt-get update \
   && apt-get install -y lsb-release \
+      git \
      apt-transport-https \
      r-base \
      libffi-dev \
@@ -36,11 +37,6 @@ RUN apt-get update \
 # install Ruby.
 RUN apt-get install -y ruby-full
 
-# install GIT from buster-backports
-RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/buster-backports.list \
-  && apt-get update \
-  && apt-get -t=buster-backports -y install git=1:2.30.*
-
 #
 # install docker tools:
 #

Docker/jenkins/Jenkins/Dockerfile

@@ -1,4 +1,4 @@
-FROM jenkins/jenkins:2.415-jdk11
+FROM jenkins/jenkins:2.434-jdk21
 
 USER root
 

files/scripts/ci-env-pool-reset.sh

@@ -29,7 +29,6 @@ source "${GEN3_HOME}/gen3/gen3setup.sh"
 cat - > jenkins-envs-services.txt <<EOF
 jenkins-blood
 jenkins-brain
-jenkins-dcp
 jenkins-genomel
 jenkins-niaid
 EOF

files/scripts/healdata/heal-cedar-data-ingest.py

@@ -85,16 +85,37 @@ def update_filter_metadata(metadata_to_update):
     ]
     # Add any new tags from advSearchFilters
     for f in metadata_to_update["advSearchFilters"]:
+        if f["key"] == "Gender":
+            continue
         tag = {"name": f["value"], "category": f["key"]}
         if tag not in tags:
             tags.append(tag)
     metadata_to_update["tags"] = tags
     return metadata_to_update
 
+
+def get_client_token(client_id: str, client_secret: str):
+    try:
+        token_url = f"http://revproxy-service/user/oauth2/token"
+        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+        params = {'grant_type': 'client_credentials'}
+        data = 'scope=openid user data'
+
+        token_result = requests.post(
+            token_url, params=params, headers=headers, data=data,
+            auth=(client_id, client_secret),
+        )
+        token =  token_result.json()["access_token"]
+    except:
+        raise Exception("Could not get token")
+    return token
+
+
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--directory", help="CEDAR Directory ID for registering ")
-parser.add_argument("--access_token", help="User access token")
+parser.add_argument("--cedar_client_id", help="The CEDAR client id")
+parser.add_argument("--cedar_client_secret", help="The CEDAR client secret")
 parser.add_argument("--hostname", help="Hostname")
 
 
@@ -103,17 +124,23 @@ def update_filter_metadata(metadata_to_update):
 if not args.directory:
     print("Directory ID is required!")
     sys.exit(1)
-if not args.access_token:
-    print("User access token is required!")
+if not args.cedar_client_id:
+    print("CEDAR client id is required!")
+    sys.exit(1)
+if not args.cedar_client_secret:
+    print("CEDAR client secret is required!")
     sys.exit(1)
 if not args.hostname:
     print("Hostname is required!")
     sys.exit(1)
 
 dir_id = args.directory
-access_token = args.access_token
+client_id = args.cedar_client_id
+client_secret = args.cedar_client_secret
 hostname = args.hostname
 
+print("Getting CEDAR client access token")
+access_token = get_client_token(client_id, client_secret)
 token_header = {"Authorization": 'bearer ' + access_token}
 
 limit = 10
@@ -169,6 +196,9 @@ def update_filter_metadata(metadata_to_update):
                     print("Metadata is already registered. Updating MDS record")
                 elif mds_res["_guid_type"] == "unregistered_discovery_metadata":
                     print("Metadata has not been registered. Registering it in MDS record")
+                else:
+                    print(f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped")
+                    continue
 
                 if "clinicaltrials_gov" in cedar_record:
                     mds_clinical_trials = cedar_record["clinicaltrials_gov"]

files/squid_whitelist/web_whitelist

@@ -76,6 +76,7 @@ go.googlesource.com
 golang.org
 gopkg.in
 grafana.com
+grafana.github.io
 http.us.debian.org
 ifconfig.io
 ingress.coralogix.us
@@ -144,6 +145,7 @@ repos.sensuapp.org
 repo.vmware.com
 repository.cloudera.com
 resource.metadatacenter.org
+rmq.n3c.ncats.io
 rules.emergingthreats.net
 rweb.quant.ku.edu
 sa-update.dnswl.org