Merge pull request #127 from chicagopcdc/pcdc_dev

Pcdc dev

.github/workflows/image_build_push.yaml

@@ -1,10 +1,10 @@
-name: Build Python Base Images and Push to Quay and ECR
+name: Build Python Base Images
 
 on: push
 
 jobs:
   python_3-9:
-    name: Python 3.9 Build and Push
+    name: Python 3.9
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"
@@ -17,7 +17,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   python_3-10:
-    name: Python 3.10 Build and Push
+    name: Python 3.10
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
@@ -30,7 +30,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   awshelper:
-    name: AwsHelper Build and Push
+    name: AwsHelper
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"

.github/workflows/image_build_push_jenkins.yaml

@@ -1,13 +1,14 @@
-name: Build Jenkins images and push to Quay
+name: Build Jenkins images
 
 on: 
   push:
     paths:
+      - .github/workflows/image_build_push_jenkins.yaml
       - Docker/jenkins/**
 
 jobs:
   jenkins:
-    name: Jenkins Build and Push
+    name: Jenkins
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins/Dockerfile"
@@ -21,7 +22,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}    
   jenkins2:
-    name: Jenkins2 Build and Push
+    name: Jenkins2
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins2/Dockerfile"
@@ -35,7 +36,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   jenkins-ci-worker:
-    name: Jenkins-CI-Worker Build and Push
+    name: Jenkins-CI-Worker
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-CI-Worker/Dockerfile"
@@ -49,7 +50,7 @@ jobs:
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
   jenkins-qa-worker:
-    name: Jenkins-QA-Worker Build and Push
+    name: Jenkins-QA-Worker
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-Worker/Dockerfile"

.github/workflows/image_build_push_squid.yaml

@@ -1,13 +1,14 @@
-name: Build Squid images and push to Quay
+name: Build Squid images
 
 on: 
   push:
     paths:
+      - .github/workflows/image_build_push_squid.yaml
       - Docker/squid/**
 
 jobs:
   squid:
-    name: Squid Build and Push
+    name: Squid image
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/squid/Dockerfile"

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-09-18T18:49:22Z",
+  "generated_at": "2023-10-26T21:32:44Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -79,7 +79,7 @@
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 122,
+        "line_number": 121,
         "type": "Secret Keyword"
       }
     ],
@@ -342,7 +342,7 @@
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 217,
+        "line_number": 191,
         "type": "Secret Keyword"
       }
     ],

Docker/jenkins/Jenkins-CI-Worker/Dockerfile

@@ -34,11 +34,10 @@ RUN set -xe && apt-get update \
      zlib1g-dev \
      zsh \
      ca-certificates-java \
-     openjdk-11-jre-headless \
   && ln -s /usr/bin/lua5.3 /usr/local/bin/lua
 
 # Use jdk11
-ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64"
+ENV JAVA_HOME="/opt/java/openjdk"
 ENV PATH="$JAVA_HOME/bin:$PATH"
 
 COPY ./certfix.sh /certfix.sh
@@ -75,7 +74,7 @@ RUN sudo install -m 0755 -d /etc/apt/keyrings \
 
 # install nodejs
 RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get update && apt-get install -y nodejs
+RUN apt-get update && apt-get install -y nodejs npm
 
 # Install postgres 13 client
 RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc| gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg && \
@@ -98,7 +97,7 @@ RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \
     sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3.8 -m pip install --upgrade pip && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade
+RUN set -xe && python3.8 -m pip install --upgrade pip setuptools && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade
 
 # install terraform
 RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \
@@ -117,6 +116,9 @@ RUN curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-ke
     && apt-get -y update \
     && apt-get -y install google-chrome-stable
 
+# data-simulator needs "/usr/share/dict/words" to generate data that isn't random strings
+RUN apt-get install --reinstall wamerican
+
 # update /etc/sudoers
 RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \
   && /bin/echo -e "\n%sudo    ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \

Docker/jenkins/Jenkins-Worker/Dockerfile

@@ -8,6 +8,7 @@ RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-ess
 
 RUN apt-get update \
   && apt-get install -y lsb-release \
+      git \
      apt-transport-https \
      r-base \
      libffi-dev \
@@ -36,11 +37,6 @@ RUN apt-get update \
 # install Ruby.
 RUN apt-get install -y ruby-full
 
-# install GIT from buster-backports
-RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/buster-backports.list \
-  && apt-get update \
-  && apt-get -t=buster-backports -y install git=1:2.30.*
-
 #
 # install docker tools:
 #

files/scripts/healdata/heal-cedar-data-ingest.py

@@ -91,10 +91,29 @@ def update_filter_metadata(metadata_to_update):
     metadata_to_update["tags"] = tags
     return metadata_to_update
 
+
+def get_client_token(client_id: str, client_secret: str):
+    try:
+        token_url = f"http://revproxy-service/user/oauth2/token"
+        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+        params = {'grant_type': 'client_credentials'}
+        data = 'scope=openid user data'
+
+        token_result = requests.post(
+            token_url, params=params, headers=headers, data=data,
+            auth=(client_id, client_secret),
+        )
+        token =  token_result.json()["access_token"]
+    except:
+        raise Exception("Could not get token")
+    return token
+
+
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--directory", help="CEDAR Directory ID for registering ")
-parser.add_argument("--access_token", help="User access token")
+parser.add_argument("--cedar_client_id", help="The CEDAR client id")
+parser.add_argument("--cedar_client_secret", help="The CEDAR client secret")
 parser.add_argument("--hostname", help="Hostname")
 
 
@@ -103,17 +122,23 @@ def update_filter_metadata(metadata_to_update):
 if not args.directory:
     print("Directory ID is required!")
     sys.exit(1)
-if not args.access_token:
-    print("User access token is required!")
+if not args.cedar_client_id:
+    print("CEDAR client id is required!")
+    sys.exit(1)
+if not args.cedar_client_secret:
+    print("CEDAR client secret is required!")
     sys.exit(1)
 if not args.hostname:
     print("Hostname is required!")
     sys.exit(1)
 
 dir_id = args.directory
-access_token = args.access_token
+client_id = args.cedar_client_id
+client_secret = args.cedar_client_secret
 hostname = args.hostname
 
+print("Getting CEDAR client access token")
+access_token = get_client_token(client_id, client_secret)
 token_header = {"Authorization": 'bearer ' + access_token}
 
 limit = 10
@@ -169,6 +194,9 @@ def update_filter_metadata(metadata_to_update):
                     print("Metadata is already registered. Updating MDS record")
                 elif mds_res["_guid_type"] == "unregistered_discovery_metadata":
                     print("Metadata has not been registered. Registering it in MDS record")
+                else:
+                    print(f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped")
+                    continue
 
                 if "clinicaltrials_gov" in cedar_record:
                     mds_clinical_trials = cedar_record["clinicaltrials_gov"]

files/squid_whitelist/web_whitelist

@@ -76,6 +76,7 @@ go.googlesource.com
 golang.org
 gopkg.in
 grafana.com
+grafana.github.io
 http.us.debian.org
 ifconfig.io
 ingress.coralogix.us
@@ -144,6 +145,7 @@ repos.sensuapp.org
 repo.vmware.com
 repository.cloudera.com
 resource.metadatacenter.org
+rmq.n3c.ncats.io
 rules.emergingthreats.net
 rweb.quant.ku.edu
 sa-update.dnswl.org

files/squid_whitelist/web_wildcard_whitelist

@@ -21,6 +21,7 @@
 .centos.org
 .ceph.com
 .chef.io
+.chordshealth.org
 .clamav.net
 .cloud.google.com
 .cloudfront.net

gen3/bin/awsrole.sh

@@ -20,6 +20,7 @@ gen3_awsrole_help() {
 # NOTE: service-account to role is 1 to 1
 #
 # @param serviceAccount to link to the role
+# @param flag (optional) - specify a flag to use a different trust policy
 #
 function gen3_awsrole_ar_policy() {
   local serviceAccount="$1"
@@ -32,6 +33,9 @@ function gen3_awsrole_ar_policy() {
   local issuer_url
   local account_id
   local vpc_name
+  shift || return 1
+  local flag=$1
+
   vpc_name="$(gen3 api environment)" || return 1
   issuer_url="$(aws eks describe-cluster \
                        --name ${vpc_name} \
@@ -42,7 +46,42 @@ function gen3_awsrole_ar_policy() {
 
   local provider_arn="arn:aws:iam::${account_id}:oidc-provider/${issuer_url}"
 
-  cat - <<EOF
+  if [[ "$flag" == "all_namespaces" ]]; then
+    # Use a trust policy that allows role to be used by multiple namespaces.
+    cat - <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    },
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "${provider_arn}"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "ForAllValues:StringLike": {
+          "${issuer_url}:aud": "sts.amazonaws.com",
+          "${issuer_url}:sub": [
+            "system:serviceaccount:*:${serviceAccount}",
+            "system:serviceaccount:argo:default"
+          ]
+        }
+      }
+    }
+  ]
+}
+EOF
+  else
+    # Use default policy
+    cat - <<EOF
 {
   "Version": "2012-10-17",
   "Statement": [
@@ -68,8 +107,10 @@ function gen3_awsrole_ar_policy() {
   ]
 }
 EOF
+  fi
 }
 
+
 #
 # Annotate the given service account with the given IAM role
 #
@@ -128,8 +169,15 @@ _tfplan_role() {
   local saName="$1"
   shift || return 1
   local namespace="$1"
+  shift || return 1
+  local flag=""
+  # Check if the "all_namespaces" flag is provided
+  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
+    flag="$2"
+    shift 2
+  fi
   local arDoc
-  arDoc="$(gen3_awsrole_ar_policy "$saName" "$namespace")" || return 1
+  arDoc="$(gen3_awsrole_ar_policy "$saName" "$namespace" "$flag")" || return 1
   gen3 workon default "${rolename}_role"
   gen3 cd
   cat << EOF > config.tfvars
@@ -199,6 +247,13 @@ EOF
     gen3_log_err $errMsg
     return 1
   fi
+  shift || return 1
+  local flag=""
+  # Check if the "all_namespaces" flag is provided
+  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
+    flag="$2"
+    shift 2
+  fi
 
   # check if the name is already used by another entity
   local entity_type
@@ -216,7 +271,7 @@ EOF
   fi
 
   TF_IN_AUTOMATION="true"
-  if ! _tfplan_role $rolename $saName $namespace; then
+  if ! _tfplan_role $rolename $saName $namespace -f $flag; then
     return 1
   fi
   if ! _tfapply_role $rolename; then