Chore/tform custom (#1426)

* chore(tform): __custom workspaces, more tests * chore(tform): more terraform 12 * chore(Jenkins): add terraform12 * chore(Jenkins): bump to nodejs 12 * test(terraform): fixes for Jenkins environment * fix(terraform): test suite * docs(README): NOOP to kick Jenkins Co-authored-by: frickjack <reubenonrye@uchicago.edu>
uc-cdis · Oct 13, 2020 · e2a5880 · e2a5880
1 parent 3d7703c
commit e2a5880
Show file tree

Hide file tree

Showing 60 changed files with 616 additions and 268 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": "^.secrets.baseline$",
+    "files": "^.secrets.baseline$|^./.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2020-10-05T14:12:56Z",
+  "generated_at": "2020-10-09T17:32:47Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -84,7 +84,7 @@
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
         "is_verified": false,
-        "line_number": 100,
+        "line_number": 103,
         "type": "Secret Keyword"
       }
     ],
@@ -452,13 +452,13 @@
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
         "is_verified": false,
-        "line_number": 538,
+        "line_number": 548,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
         "is_verified": false,
-        "line_number": 558,
+        "line_number": 568,
         "type": "Secret Keyword"
       }
     ],
@@ -544,6 +544,32 @@
         "type": "Secret Keyword"
       }
     ],
+    "gen3/test/terraformTest.sh": [
+      {
+        "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "is_verified": false,
+        "line_number": 135,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
+        "is_verified": false,
+        "line_number": 151,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
+        "is_verified": false,
+        "line_number": 155,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
+        "is_verified": false,
+        "line_number": 155,
+        "type": "Secret Keyword"
+      }
+    ],
     "kube/services/jobs/indexd-authz-job.yaml": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",

diff --git a/Docker/Jenkins/Dockerfile b/Docker/Jenkins/Dockerfile
@@ -1,4 +1,4 @@
-FROM jenkins/jenkins:2.253
+FROM jenkins/jenkins:2.260
 
 USER root
 
@@ -77,7 +77,7 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
 
 
 # install nodejs
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
 RUN apt-get update && apt-get install -y nodejs
 
 # install chrome (supports headless mode)
@@ -91,6 +91,9 @@ RUN set -xe \
 RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.14/terraform_0.11.14_linux_amd64.zip \
    && unzip /tmp/terraform.zip -d /usr/local/bin && /bin/rm /tmp/terraform.zip
 
+RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.12.29/terraform_0.12.29_linux_amd64.zip \
+   && unzip /tmp/terraform.zip -d /tmp && mv /tmp/terraform /usr/local/bin/terraform12 && /bin/rm /tmp/terraform.zip
+
 # install packer
 RUN curl -o /tmp/packer.zip https://releases.hashicorp.com/packer/1.5.1/packer_1.5.1_linux_amd64.zip
 RUN unzip /tmp/packer.zip -d /usr/local/bin; /bin/rm /tmp/packer.zip

diff --git a/README.md b/README.md
@@ -235,4 +235,3 @@ We have two automation scripts:
 The latest versions of the customized ubuntu 16.04 AMI's required by the terraform automation
 are published as public images under the AWS `cdis-test` account.
 Build new AMIs using [images](https://github.com/uc-cdis/images).
-
diff --git a/doc/terraform/workon.md b/doc/terraform/workon.md
@@ -19,6 +19,29 @@ USE: gen3 workon PROFILE_NAME WORKSPACE_NAME
 
 The WORKSPACE_NAME may specify its terraform script folder with a `__FOLDER` suffix. For example, `gen3 workon commons labsetup__csoc_qualys_vm` will run the terraform scripts under `tf_files/aws/csoc_qualys_vm`.
 
+The WORKSPACE_NAME may specify its terraform script folder as `__custom` to
+indicate that the workspace will define its own script.  For example:
+
+```
+  gen3 workon cdistest reubenbucket__custom
+  gen3 cd
+  cat - > bucket.tf <<EOM
+provider "aws" {}
+
+module "s3_bucket" {
+  bucket_name       = "frickjack-crazy-test"
+  environment       = "qaplanetv1"
+  source            = "../../../../../cloud-automation/tf_files/aws/modules/s3-bucket"
+  cloud_trail_count = "0"
+}
+EOM
+
+# run workon to re-init local modules
+gen3 workon . .
+gen3 tfplan
+gen3 tfapply
+```
+
 ## Example
 
 * `gen3 workon cdistest devplanetv1`

diff --git a/gen3/bin/kube-setup-workvm.sh b/gen3/bin/kube-setup-workvm.sh
@@ -91,7 +91,7 @@ if sudo -n true > /dev/null 2>&1 && [[ $(uname -s) == "Linux" ]]; then
 
     install_terraform12() {
       mkdir "${XDG_RUNTIME_DIR}/t12"
-      curl -o "${XDG_RUNTIME_DIR}/t12/terraform12.zip" https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_linux_amd64.zip
+      curl -o "${XDG_RUNTIME_DIR}/t12/terraform12.zip" https://releases.hashicorp.com/terraform/0.12.29/terraform_0.12.29_linux_amd64.zip
       sudo /bin/rm -rf /usr/local/bin/terraform12 > /dev/null 2>&1 || true
       unzip "${XDG_RUNTIME_DIR}/t12/terraform12.zip" -d "${XDG_RUNTIME_DIR}/t12";
       sudo cp "${XDG_RUNTIME_DIR}/t12/terraform" "/usr/local/bin/terraform12"
@@ -110,7 +110,7 @@ if sudo -n true > /dev/null 2>&1 && [[ $(uname -s) == "Linux" ]]; then
       install_terraform12  
     else
       T12_VERSION=$(terraform12 --version | head -1 | awk '{ print $2 }' | sed 's/^[^0-9]*//')
-      if ! semver_ge "$T12_VERSION" "0.12.24"; then
+      if ! semver_ge "$T12_VERSION" "0.12.29"; then
         install_terraform
       fi
     fi
@@ -149,7 +149,7 @@ EOM
     )
   fi
   if ! which packer > /dev/null 2>&1; then
-    curl -o "${XDG_RUNTIME_DIR}/packer.zip" https://releases.hashicorp.com/packer/1.2.1/packer_1.2.1_linux_amd64.zip
+    curl -o "${XDG_RUNTIME_DIR}/packer.zip" https://releases.hashicorp.com/packer/1.5.1/packer_1.5.1_linux_amd64.zip
     sudo unzip "${XDG_RUNTIME_DIR}/packer.zip" -d /usr/local/bin
     /bin/rm "${XDG_RUNTIME_DIR}/packer.zip"
   fi

diff --git a/gen3/bin/tfapply.sh b/gen3/bin/tfapply.sh
@@ -1,40 +1,34 @@
-help() {
-  cat - <<EOM
-  gen3 tfapply:
-    Run 'terraform apply' in the current workspace, and backup config.tfvars, backend.tfvars, and README.md.  
-    A typical command line is:
-       terraform apply plan.terraform
-EOM
-  return 0
-}
-
-
 source "$GEN3_HOME/gen3/lib/utils.sh"
 gen3_load "gen3/lib/terraform"
 
-cd $GEN3_WORKDIR
+cd "$GEN3_WORKDIR"
 if [[ ! -f plan.terraform ]]; then
-  echo "plan.terraform does not exist in workspace - run 'gen3 tfplan'"
+  gen3_log_err "plan.terraform does not exist in workspace - run 'gen3 tfplan'"
   exit 1
 fi
 
-$GEN3_DRY_RUN && "Running in DRY_RUN mode ..."
+$GEN3_DRY_RUN && gen3_log_info "Running in DRY_RUN mode ..."
 
-echo "Running: terraform apply plan.terraform"
+gen3_log_info "Running: terraform apply plan.terraform"
 if ! ($GEN3_DRY_RUN || gen3_terraform apply plan.terraform); then
-  echo "apply failed, bailing out"
+  gen3_log_err "apply failed, bailing out"
   exit 1
 fi
 
+if [[ -d .git ]] && ! $GEN3_DRY_RUN; then
+  git add .
+  git commit -n -m 'pre-apply auto-commit' 1>&2
+fi
+
 dryRunFlag=""
 if $GEN3_DRY_RUN; then
   dryRunFlag="--dryrun"
 fi
 if [[ "$GEN3_FLAVOR" == "AWS" ]]; then
-  echo "Backing up files to s3"
+  gen3_log_info "Backing up files to s3"
   for fileName in config.tfvars backend.tfvars README.md; do
     s3Path="s3://${GEN3_S3_BUCKET}/${GEN3_WORKSPACE}/$fileName"
-    echo "Backing up $fileName to $s3Path"
+    gen3_log_info "Backing up $fileName to $s3Path"
     gen3_aws_run aws s3 cp $dryRunFlag --sse AES256 "$fileName" "$s3Path"
   done
 fi
diff --git a/gen3/bin/workon.sh b/gen3/bin/workon.sh
@@ -2,52 +2,13 @@
 # Helper script for 'gen3 workon' - see ../README.md and ../gen3setup.sh
 #
 
-help() {
-  cat - <<EOM
-  Use: gen3 workon aws-profile vpc-name
-     Prepares a local workspace to run terraform and other devops tools.
-EOM
-  return 0
-}
-
 source "$GEN3_HOME/gen3/lib/utils.sh"
 gen3_load "gen3/lib/aws"
 gen3_load "gen3/lib/gcp"
 gen3_load "gen3/lib/onprem"
 gen3_load "gen3/lib/terraform"
 
-#
-# Create any missing files
-#
-mkdir -p -m 0700 "$GEN3_WORKDIR/backups"
-chmod 0700 "$GEN3_WORKDIR"
-
-if [[ ! -f "$GEN3_WORKDIR/root.tf" ]]; then
-  # Note: do not use `` in heredoc!
-  echo "Creating $GEN3_WORKDIR/root.tf"
-  if [[ "$GEN3_FLAVOR" == "AWS" ]]; then
-    cat - > "$GEN3_WORKDIR/root.tf" <<EOM
-#
-# THIS IS AN AUTOGENERATED FILE (by gen3)
-# root.tf is required for *terraform output*, *terraform taint*, etc
-# @see https://github.com/hashicorp/terraform/issues/15761
-#
-terraform {
-    backend "s3" {
-        encrypt = "true"
-    }
-}
-EOM
-  else
-    cat - > "$GEN3_WORKDIR/root.tf" <<EOM
-#
-# THIS IS AN AUTOGENERATED FILE (by gen3)
-# root.tf is required for *terraform output*, *terraform taint*, etc
-# @see https://github.com/hashicorp/terraform/issues/15761
-#
-EOM
-  fi
-fi
+# lib -------------------
 
 #
 # Sync the given file with S3.
@@ -65,7 +26,7 @@ refreshFromBackend() {
   fi
   filePath="${GEN3_WORKDIR}/$fileName"
   if [[ -f $filePath ]]; then
-    echo "Ignoring S3 refresh for file that already exists: $fileName"
+    gen3_log_info "Ignoring S3 refresh for file that already exists: $fileName"
     return 1
   fi
   if [[ "$GEN3_FLAVOR" != "AWS" ]]; then
@@ -75,20 +36,54 @@ refreshFromBackend() {
   s3Path="s3://${GEN3_S3_BUCKET}/${GEN3_WORKSPACE}/${fileName}"
   gen3_aws_run aws s3 cp "$s3Path" "$filePath" > /dev/null 2>&1
   if [[ ! -f "$filePath" ]]; then
-    echo "No data at $s3Path"
+    gen3_log_info "No data at $s3Path"
     return 1
   fi
   return 0
 }
 
+# main -----------------------
+
+#
+# Create any missing files
+#
+mkdir -p -m 0700 "$GEN3_WORKDIR/backups"
+chmod 0700 "$GEN3_WORKDIR"
+
+if [[ ! -f "$GEN3_WORKDIR/root.tf" ]]; then
+  # Note: do not use `` in heredoc!
+  gen3_log_info "Creating $GEN3_WORKDIR/root.tf"
+  if [[ "$GEN3_FLAVOR" == "AWS" ]]; then
+    cat - > "$GEN3_WORKDIR/root.tf" <<EOM
+#
+# THIS IS AN AUTOGENERATED FILE (by gen3)
+# root.tf is required for *terraform output*, *terraform taint*, etc
+# @see https://github.com/hashicorp/terraform/issues/15761
+#
+terraform {
+    backend "s3" {
+        encrypt = "true"
+    }
+}
+EOM
+  else
+    cat - > "$GEN3_WORKDIR/root.tf" <<EOM
+#
+# THIS IS AN AUTOGENERATED FILE (by gen3)
+# root.tf is required for *terraform output*, *terraform taint*, etc
+# @see https://github.com/hashicorp/terraform/issues/15761
+#
+EOM
+  fi
+fi
 
 for fileName in config.tfvars backend.tfvars README.md; do
   filePath="${GEN3_WORKDIR}/$fileName"
   if [[ ! -f "$filePath" ]]; then
     refreshFromBackend "$fileName"
     if [[ ! -f "$filePath" ]]; then
-      echo "Variables not configured at $filePath"
-      echo "Setting up initial contents - customize before running terraform"
+      gen3_log_info "Variables not configured at $filePath"
+      gen3_log_info "Setting up initial contents - customize before running terraform"
       # Run the function that corresponds to the profile flavor (AWS, GCP, ...) and $fileName
       "gen3_${GEN3_FLAVOR}.$fileName" > "$filePath"
     fi
@@ -98,13 +93,13 @@ done
 cd "$GEN3_WORKDIR"
 bucketCheckFlag=".tmp_bucketcheckflag2"
 if [[ ! -f "$bucketCheckFlag" && "$GEN3_FLAVOR" == "AWS" ]]; then
-  echo "initializing terraform"
-  echo "checking if $GEN3_S3_BUCKET bucket exists"
+  gen3_log_info "initializing terraform"
+  gen3_log_info "checking if $GEN3_S3_BUCKET bucket exists"
   if ! gen3_aws_run aws s3 ls "s3://$GEN3_S3_BUCKET" > /dev/null 2>&1; then
-    echo "Creating $GEN3_S3_BUCKET bucket"
-    echo "NOTE: please verify that aws profile region matches backend.tfvars region:"
-    echo "  aws profile region: $(aws configure get $GEN3_PROFILE.region)"
-    echo "  terraform backend region: $(cat *backend.tfvars | grep region)"
+    gen3_log_info "Creating $GEN3_S3_BUCKET bucket"
+    gen3_log_info "NOTE: please verify that aws profile region matches backend.tfvars region:"
+    gen3_log_info "  aws profile region: $(aws configure get $GEN3_PROFILE.region)"
+    gen3_log_info "  terraform backend region: $(cat *backend.tfvars | grep region)"
 
     S3_POLICY=$(cat - <<EOM
   {
@@ -128,5 +123,38 @@ EOM
   fi
 fi
 
-echo "Running: terraform init --backend-config ./backend.tfvars $GEN3_TFSCRIPT_FOLDER/"
+# setup git
+(
+    cd "${GEN3_WORKDIR}/"
+    if [[ ! -d ".git" ]]; then
+      git init .
+      cat > .gitignore <<EOM
+.*
+*.log
+*.bak
+*~
+
+!.gitignore
+EOM
+      git add .
+      git commit -n -m 'initial'
+    fi
+)
+
+if [[ "$GEN3_WORKSPACE" =~ __custom$ ]]; then
+  ( # pin terraform version
+    cd "${GEN3_WORKDIR}/"
+    if [[ ! -f "manifest.json" ]]; then
+      cat - > manifest.json <<EOM
+{
+  "terraform": {
+    "module_version" : "0.12"
+  }
+}
+EOM
+    fi
+  )
+fi
+
+gen3_log_info "Running: terraform init --backend-config ./backend.tfvars $GEN3_TFSCRIPT_FOLDER/ in $(pwd)"
 gen3_terraform init --backend-config ./backend.tfvars "$GEN3_TFSCRIPT_FOLDER/"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -235,4 +235,3 @@ We have two automation scripts:
		The latest versions of the customized ubuntu 16.04 AMI's required by the terraform automation
		are published as public images under the AWS `cdis-test` account.
		Build new AMIs using [images](https://github.com/uc-cdis/images).