diff --git a/.secrets.baseline b/.secrets.baseline index def2ce187..12842d005 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-07-31T16:54:24Z", + "generated_at": "2023-08-10T17:51:06Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -78,7 +78,7 @@ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_verified": false, - "line_number": 115, + "line_number": 122, "type": "Secret Keyword" } ], diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index 671cd2e02..40fd08fa3 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -54,6 +54,13 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \ google-cloud-sdk-cbt \ kubectl +# install go - https://go.dev/doc/install +RUN wget https://go.dev/dl/go1.21.0.linux-amd64.tar.gz \ + && rm -rf /usr/local/go \ + && tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz +ENV PATH="$PATH:/usr/local/go/bin" +RUN go version + # # install docker tools: # diff --git a/Docker/jenkins/Jenkins-Worker/Dockerfile b/Docker/jenkins/Jenkins-Worker/Dockerfile index 088186b04..c31e54923 100644 --- a/Docker/jenkins/Jenkins-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-Worker/Dockerfile @@ -53,6 +53,12 @@ RUN sudo install -m 0755 -d /etc/apt/keyrings \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ && apt-get update && apt-get install -y docker-ce +# install k6 to run load tests +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 \ + && echo "deb https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list \ + && apt-get update \ + && apt-get install k6 + # install xk6-browser RUN cd /opt && wget --quiet https://github.com/grafana/xk6-browser/releases/download/v0.3.0/xk6-browser-v0.3.0-linux-amd64.tar.gz \ && tar -xvzf /opt/xk6-browser-v0.3.0-linux-amd64.tar.gz diff --git a/apis_configs/pcdcanalysistools_settings.py b/apis_configs/pcdcanalysistools_settings.py index b6f366d8e..2fc14976e 100644 --- a/apis_configs/pcdcanalysistools_settings.py +++ b/apis_configs/pcdcanalysistools_settings.py @@ -84,7 +84,7 @@ def load_json(file_name): config['PATH_TO_SCHEMA_DIR'] = environ.get('PATH_TO_SCHEMA_DIR') config['SURVIVAL'] = { - 'consortium': ["INSTRuCT", "INRG"], + 'consortium': ["INSTRuCT", "INRG", "MaGIC", "NODAL"], 'excluded_variables': [ { 'label': 'Data Contributor', diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index c36194765..349d1e022 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -10,6 +10,7 @@ api.monqcle.com biodata-integration-tests.net marketing.biorender.com clinicaltrials.gov +charts.bitnami.com ctds-planx.atlassian.net data.cityofchicago.org dataguids.org @@ -69,6 +70,7 @@ ftp.ussg.iu.edu fmwww.bc.edu gcr.io get.helm.sh +ghcr.io git.io go.googlesource.com golang.org @@ -123,6 +125,7 @@ orcid.org pgp.mit.edu ppa.launchpad.net prometheus-community.github.io +proxy.golang.org public.ecr.aws pubmirrors.dal.corespace.com reflector.westga.edu @@ -138,6 +141,7 @@ repo.dimenoc.com repos.mia.quadranet.com repos.redrockhost.com repos.sensuapp.org +repo.vmware.com repository.cloudera.com resource.metadatacenter.org rules.emergingthreats.net diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist index 7e13d7d07..3c96873f1 100644 --- a/files/squid_whitelist/web_wildcard_whitelist +++ b/files/squid_whitelist/web_wildcard_whitelist @@ -38,6 +38,7 @@ .dph.illinois.gov .elasticsearch.org .erlang-solutions.com +.external-secrets.io .extjs.com .fedoraproject.org .gen3.org diff --git a/gen3/bin/dbbackup.sh b/gen3/bin/dbbackup.sh new file mode 100644 index 000000000..29f267221 --- /dev/null +++ b/gen3/bin/dbbackup.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +#################################################################################################### +# Script: dbdump.sh +# +# Description: +# This script facilitates the management of database backups within the gen3 environment. It is +# equipped to establish policies, service accounts, roles, and S3 buckets. Depending on the +# command provided, it will either initiate a database dump or perform a restore. +# +# Usage: +# gen3 dbbackup [dump|restore] +# +# dump - Initiates a database dump, creating the essential AWS resources if they are absent. +# The dump operation is intended to be executed from the namespace/commons that requires +# the backup. +# restore - Initiates a database restore, creating the essential AWS resources if they are absent. +# The restore operation is meant to be executed in the target namespace, where the backup +# needs to be restored. +# +# Notes: +# This script extensively utilizes the AWS CLI and the gen3 CLI. Proper functioning demands a +# configured gen3 environment and the availability of the necessary CLI tools. +# +#################################################################################################### + +# Exit on error +#set -e + +# Print commands before executing +#set -x + +#trap 'echo "Error at Line $LINENO"' ERR + +source "${GEN3_HOME}/gen3/lib/utils.sh" +gen3_load "gen3/lib/kube-setup-init" + +policy_name="bucket_reader_writer_gen3_db_backup" +account_id=$(aws sts get-caller-identity --query "Account" --output text) +vpc_name="$(gen3 api environment)" +namespace="$(gen3 db namespace)" +sa_name="dbbackup-sa" +bucket_name="gen3-db-backups-${account_id}" + +gen3_log_info "policy_name: $policy_name" +gen3_log_info "account_id: $account_id" +gen3_log_info "vpc_name: $vpc_name" +gen3_log_info "namespace: $namespace" +gen3_log_info "sa_name: $sa_name" +gen3_log_info "bucket_name: $bucket_name" + + +# Create an S3 access policy if it doesn't exist +create_policy() { + # Check if policy exists + if ! aws iam list-policies --query "Policies[?PolicyName == '$policy_name'] | [0].Arn" --output text | grep -q "arn:aws:iam"; then + # Create the S3 access policy - policy document + access_policy=$(cat <<-EOM +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::gen3-db-backups-*" + ] + } + ] +} +EOM + ) + + # Create the S3 access policy from the policy document + policy_arn=$(aws iam create-policy --policy-name "$policy_name" --policy-document "$access_policy" --query "Policy.Arn" --output text) + gen3_log_info "policy_arn: $policy_arn" + else + gen3_log_info "Policy $policy_name already exists, skipping policy creation." + policy_arn=$(aws iam list-policies --query "Policies[?PolicyName == '$policy_name'] | [0].Arn" --output text | grep "arn:aws:iam" | head -n 1) + gen3_log_info "policy_arn: $policy_arn" + fi +} + + +# Create or update the Service Account and its corresponding IAM Role +create_service_account_and_role() { + cluster_arn=$(kubectl config current-context) + eks_cluster=$(echo "$cluster_arn" | awk -F'/' '{print $2}') + oidc_url=$(aws eks describe-cluster --name $eks_cluster --query 'cluster.identity.oidc.issuer' --output text | sed -e 's/^https:\/\///') + role_name="${vpc_name}-${namespace}-${sa_name}-role" + role_arn="arn:aws:iam::${account_id}:role/${role_name}" + local trust_policy=$(mktemp -p "$XDG_RUNTIME_DIR" "tmp_policy.XXXXXX") + gen3_log_info "trust_policy: $trust_policy" + gen3_log_info "eks_cluster: $eks_cluster" + gen3_log_info "oidc_url: $oidc_url" + gen3_log_info "role_name: $role_name" + + + cat > ${trust_policy} <&1; then + gen3_log_info "Updating existing role: $role_name" + aws iam update-assume-role-policy --role-name $role_name --policy-document "file://$trust_policy" + else + gen3_log_info "Creating new role: $role_name" + aws iam create-role --role-name $role_name --assume-role-policy-document "file://$trust_policy" + fi + + # Attach the policy to the IAM role + aws iam attach-role-policy --role-name $role_name --policy-arn $policy_arn + + # Create the Kubernetes service account if it doesn't exist + if ! kubectl get serviceaccount -n $namespace $sa_name 2>&1; then + kubectl create serviceaccount -n $namespace $sa_name + fi + # Annotate the KSA with the IAM role ARN + gen3_log_info "Annotating Service Account with IAM role ARN" + kubectl annotate serviceaccount -n ${namespace} ${sa_name} eks.amazonaws.com/role-arn=${role_arn} --overwrite + +} + +# Create an S3 bucket if it doesn't exist +create_s3_bucket() { + # Check if bucket already exists + if aws s3 ls "s3://$bucket_name" 2>&1 | grep -q 'NoSuchBucket'; then + gen3_log_info "Bucket does not exist, creating..." + aws s3 mb "s3://$bucket_name" + else + gen3_log_info "Bucket $bucket_name already exists, skipping bucket creation." + fi +} + + +# Function to trigger the database backup job +db_dump() { + gen3 job run psql-db-prep-dump +} + + +# Function to trigger the database backup restore job +db_restore() { + gen3 job run psql-db-prep-restore +} + + +# main function to determine whether dump or restore +main() { + case "$1" in + dump) + gen3_log_info "Triggering database dump..." + create_policy + create_service_account_and_role + create_s3_bucket + db_dump + ;; + restore) + gen3_log_info "Triggering database restore..." + create_policy + create_service_account_and_role + create_s3_bucket + db_restore + ;; + *) + echo "Invalid command. Usage: gen3 dbbackup [dump|restore]" + return 1 + ;; + esac +} + +main "$1" diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh index 57ae1183d..8e7302fc5 100644 --- a/gen3/bin/kube-roll-all.sh +++ b/gen3/bin/kube-roll-all.sh @@ -257,6 +257,18 @@ else gen3_log_info "not deploying dicom-viewer - no manifest entry for '.versions[\"dicom-viewer\"]'" fi +if g3k_manifest_lookup '.versions["ohdsi-atlas"]' && g3k_manifest_lookup '.versions["ohdsi-webapi"]' 2> /dev/null; then + gen3 kube-setup-ohdsi & +else + gen3_log_info "not deploying OHDSI tools - no manifest entry for '.versions[\"ohdsi-atlas\"]' and '.versions[\"ohdsi-webapi\"]'" +fi + +if g3k_manifest_lookup '.versions["cohort-middleware"]' 2> /dev/null; then + gen3 kube-setup-cohort-middleware +else + gen3_log_info "not deploying cohort-middleware - no manifest entry for .versions[\"cohort-middleware\"]" +fi + gen3 kube-setup-revproxy if [[ "$GEN3_ROLL_FAST" != "true" ]]; then @@ -348,18 +360,6 @@ else gen3_log_info "not deploying argo-wrapper - no manifest entry for '.versions[\"argo-wrapper\"]'" fi -if g3k_manifest_lookup '.versions["cohort-middleware"]' 2> /dev/null; then - gen3 roll cohort-middleware & -else - gen3_log_info "not deploying cohort-middleware - no manifest entry for '.versions[\"cohort-middleware\"]'" -fi - -if g3k_manifest_lookup '.versions["ohdsi-atlas"]' && g3k_manifest_lookup '.versions["ohdsi-webapi"]' 2> /dev/null; then - gen3 kube-setup-ohdsi & -else - gen3_log_info "not deploying OHDSI tools - no manifest entry for '.versions[\"ohdsi-atlas\"]' and '.versions[\"ohdsi-webapi\"]'" -fi - gen3_log_info "enable network policy" gen3 kube-setup-networkpolicy "enable" || true & diff --git a/gen3/bin/kube-setup-ambassador.sh b/gen3/bin/kube-setup-ambassador.sh index 0f4e0be28..5f92af5cc 100644 --- a/gen3/bin/kube-setup-ambassador.sh +++ b/gen3/bin/kube-setup-ambassador.sh @@ -25,7 +25,6 @@ deploy_api_gateway() { return 0 fi gen3 roll ambassador-gen3 - g3k_kv_filter "${GEN3_HOME}/kube/services/ambassador-gen3/ambassador-gen3-service-elb.yaml" GEN3_ARN "$(g3kubectl get configmap global --output=jsonpath='{.data.revproxy_arn}')" | g3kubectl apply -f - local luaYamlTemp="$(mktemp "$XDG_RUNTIME_DIR/lua.yaml.XXXXXX")" cat - > "$luaYamlTemp" < "${database}.sql" + + if [ $? -eq 0 ] && [ -f "${database}.sql" ]; then + gen3_log_info "Uploading backup file ${database}.sql to s3://${bucket_name}/${s3_dir}/${database}.sql" + aws s3 cp "${database}.sql" "s3://${bucket_name}/${s3_dir}/${database}.sql" + + if [ $? -eq 0 ]; then + gen3_log_info "Successfully uploaded ${database}.sql to S3" + else + gen3_log_err "Failed to upload ${database}.sql to S3" + fi + gen3_log_info "Deleting temporary backup file ${database}.sql" + rm -f "${database}.sql" + else + gen3_log_err "Backup operation failed for ${database}" + rm -f "${database}.sql" + fi + done + sleep 600 + restartPolicy: Never + diff --git a/kube/services/jobs/psql-db-prep-restore-job.yaml b/kube/services/jobs/psql-db-prep-restore-job.yaml new file mode 100644 index 000000000..710e6f4f1 --- /dev/null +++ b/kube/services/jobs/psql-db-prep-restore-job.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: psql-db-prep-restore +spec: + template: + metadata: + labels: + app: gen3job + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - weight: 99 + preference: + matchExpressions: + - key: eks.amazonaws.com/capacityType + operator: In + values: + - ONDEMAND + serviceAccountName: dbbackup-sa + containers: + - name: pgrestore + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + env: + - name: gen3Env + valueFrom: + configMapKeyRef: + name: global + key: environment + - name: JENKINS_HOME + value: "devterm" + - name: GEN3_HOME + value: /home/ubuntu/cloud-automation + command: [ "/bin/bash" ] + args: + - "-c" + - | + source "${GEN3_HOME}/gen3/lib/utils.sh" + gen3_load "gen3/gen3setup" + account_id=$(aws sts get-caller-identity --query "Account" --output text) + default_bucket_name="gen3-db-backups-${account_id}" + default_databases=("indexd" "sheepdog" "metadata") + backup_directories=$(aws s3 ls "s3://${default_bucket_name}/") + newest_directory=$(echo "$backup_directories" | awk '/PRE/ {if ($2 > max) max = $2} END {print max}') + databases=("${default_databases[@]}") + bucket_name=$default_bucket_name + namespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + date_str=$(date -u +%y%m%d_%H%M%S) + gen3_log_info "Database backup location in S3: ${bucket_name}/${newest_directory}" + gen3_log_info "namespace: $namespace \n\n" + + for database in "${databases[@]}"; do + gen3_log_info "Downloading database backup file s3://${default_bucket_name}/${newest_directory}${database}.sql" + aws s3 cp "s3://${default_bucket_name}/${newest_directory}${database}.sql" "${database}.sql" + server=$(gen3 db creds "$database" | jq -r '.g3FarmServer') + username=$(gen3 db creds "$database" | jq -r '.db_username') + db_name="${namespace}_${database}_${date_str}" + if [[ -z "$server" || -z "$username" ]]; then + gen3_log_info "Error: Unable to extract server name or username." + return 1 + fi + gen3 psql $database -c "create database $db_name;" 2>&1 | grep -q "permission denied" + if [ $? -eq 0 ]; then + gen3_log_info "User does not have permission to create database. Granting required permission..." + gen3 psql $server -c "alter user $username createdb;" + gen3 psql $database -c "create database $db_name;" + if [ $? -eq 0 ]; then + gen3_log_info "Database $db_name created successfully!" + else + gen3_log_info "Error creating database $db_name after granting permission." + fi + else + gen3_log_info "Database $db_name created successfully!" + fi + gen3_log_info "Starting database restore for ${database} to database $db_name" + gen3 psql "$database" -d "$db_name" -f "${database}.sql" 1>&2 + gen3_log_info "cleanup temporary backup file ${database}.sql \n\n\n" + done + sleep 600 + restartPolicy: Never diff --git a/kube/services/monitoring/values.yaml b/kube/services/monitoring/values.yaml index ffdf92bd9..d93e5098a 100644 --- a/kube/services/monitoring/values.yaml +++ b/kube/services/monitoring/values.yaml @@ -1540,6 +1540,15 @@ prometheus-node-exporter: - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ service: portName: http-metrics + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "eks.amazonaws.com/compute-type" + operator: NotIn + values: + - fargate prometheus: monitor: enabled: true