responded to github comments

build_docker.sh

@@ -93,7 +93,7 @@ if [ -n "$STAGING_DIR" ]; then
     set +e
     rm -Rf ${STAGING_DIR}/${STAGING_CLONE_DIR}
     set -e
-    GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/${REPO}/${PROJECT}.git ${STAGING_DIR}/${STAGING_CLONE_DIR}
+    GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/${REPO}/${PROJECT}.git ${STAGING_CLONE_DIR}
     cd ${STAGING_DIR}/${STAGING_CLONE_DIR}
     STAGING_ABSOLUTE_PATH=$(pwd)
 

build_docker_remote.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Build (and optionally push) a GATK docker image.
+# Build (and optionally push) a GATK docker image to GCR using Google Cloud Build. Images are built in the cloud rather than locally. Pushing to dockerhub is not supported by this script.
 #
 # If you are pushing an image to our release repositories, be sure that you've followed
 # the setup instructions here:
@@ -14,14 +14,13 @@ set -e
 
 REPO=broadinstitute
 PROJECT=gatk
-REPO_PRJ=${REPO}/${PROJECT}
 GCR_REPO="us.gcr.io/broad-dsde-methods/broad-gatk-snapshots/gatk-remote-builds"
 STAGING_CLONE_DIR=${PROJECT}_staging_temp
 
 #################################################
 # Parsing arguments
 #################################################
-while getopts "e:pslrud:t:" option; do
+while getopts "e:sd:t:" option; do
 	case "$option" in
 		e) GITHUB_TAG="$OPTARG" ;;
 		s) IS_HASH=true ;;
@@ -34,10 +33,9 @@ if [ -z "$GITHUB_TAG" ]; then
 	printf "Option -e requires an argument.\n \
 Usage: %s: -e <GITHUB_TAG> [-psl] \n \
 where <GITHUB_TAG> is the github tag (or hash when -s is used) to use in building the docker image\n \
-(e.g. bash build_docker.sh -e 1.0.0.0-alpha1.2.1)\n \
+(e.g. bash build_docker_remote.sh -e 4.2.6.1 )\n \
 Optional arguments:  \n \
 -s \t The GITHUB_TAG (-e parameter) is actually a github hash, not tag.  git hashes cannot be pushed as latest, so -l is implied.  \n \
--l \t Do not also push the image to the 'latest' tag. \n \
 -d <STAGING_DIR> \t staging directory to grab code from repo and build the docker image.  If unspecified, then use whatever is in current dir (do not go to the repo).  NEVER SPECIFY YOUR WORKING DIR \n \
 -t <IMAGE_TAG>\t  The tag to assign image once it is finished constructing.  NOTE: currently this MUST be on either GCR or the Google Artifact Registry. \n" $0
 	exit 1
@@ -46,13 +44,11 @@ fi
 # Output the parameters
 echo -e "\n"
 echo -e "github tag/hash: ${GITHUB_TAG}"
-echo -e "container registry repo, project, and tag: ${REPO_PRJ}:${GITHUB_TAG}\n\n"
+echo -e "github project: ${REPO_PRJ}:${GITHUB_TAG}\n\n"
 echo "Other options (Blank is false)"
 echo "---------------"
 echo "This is a git hash: ${IS_HASH}"
-echo "Push to dockerhub: ${IS_PUSH}"
 echo "Staging directory: ${STAGING_DIR}"
-echo -e "Fetch from this remote path: ${PULL_REQUEST_NUMBER}\n\n\n"
 
 ORIGINAL_WORKING_DIRECTORY=$(pwd)
 
@@ -68,22 +64,17 @@ if [ -n "$STAGING_DIR" ]; then
     set +e
     rm -Rf ${STAGING_DIR}/${STAGING_CLONE_DIR}
     set -e
-    GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/${REPO}/${PROJECT}.git ${STAGING_DIR}/${STAGING_CLONE_DIR}
+    GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/${REPO}/${PROJECT}.git ${STAGING_CLONE_DIR}
     cd ${STAGING_DIR}/${STAGING_CLONE_DIR}
     STAGING_ABSOLUTE_PATH=$(pwd)
 
     echo "Now in $(pwd)"
-    if [ ${PULL_REQUEST_NUMBER} ]; then
-        GIT_FETCH_COMMAND="git fetch origin +refs/pull/${PULL_REQUEST_NUMBER}/merge"
-        echo "${GIT_FETCH_COMMAND}"
-        ${GIT_FETCH_COMMAND}
-    fi
     GIT_CHECKOUT_COMMAND="git checkout ${GITHUB_DIR}${GITHUB_TAG}"
     echo "${GIT_CHECKOUT_COMMAND}"
     ${GIT_CHECKOUT_COMMAND}
 fi
 
-GIT_HASH_FOR_TAG=$(git describe --tags)
+GIT_HASH_FOR_TAG=$(git describe --tags --long)
 
 ## generate the tag if it wasn't explicitly specified
 if [ -z "$DOCKER_IMAGE_TAG" ]; then
@@ -94,10 +85,13 @@ echo "Building image with the tag ${DOCKER_IMAGE_TAG}..."
 
 SUBMIT_COMMAND="gcloud builds submit --tag ${DOCKER_IMAGE_TAG} --timeout=24h --machine-type n1_highcpu_8"
 echo "running the following gcloud command: ${SUBMIT_COMMAND}"
+## We need to override the default .gcloudignore to preserve the .git directory which we need in order to download LFS files in the remote build.
 echo -n "" >> .gcloudignore
 ${SUBMIT_COMMAND}
 
 cd ${ORIGINAL_WORKING_DIRECTORY}
 if [ -n "$STAGING_DIR" ] ; then
     rm -Rf ${STAGING_DIR}/${STAGING_CLONE_DIR}
 fi
+
+exit 0

scripts/docker/README.md

@@ -23,7 +23,7 @@ This repo contains the scripts for creating and pushing two docker images:
 
 ``scripts/docker/gatkbase/build_docker_base.sh`` is a script to create the gatkbase docker image. 
 ``build_docker.sh`` is a script to create the full gatk4 docker image.
-``build_docker_remote.sh`` is a script to create the full gatk4 docker image using google cloud build remotely. NOTE: this requires the user first specify their project with the command `gcloud config set project <PROJECT>` to a project that has access to google cloud build. 
+``build_docker_remote.sh`` is a script to create the full gatk4 docker image using google cloud build remotely. This is useful if you can't build the docker image locally (if you have an M1 Macbook) NOTE: this requires the user first specify their project with the command `gcloud config set project <PROJECT>` to a project that has access to google cloud build. 
 
 ## GATK4 Docker image
 
@@ -48,7 +48,7 @@ sudo bash build_docker.sh -e ${GITHUB_TAG} -p -d ${STAGING_DIR}
 
 #### Create GATK4 docker image and upload into GCR
 
-This will attempt to build and automatically upload the docker image with a tag following this format: `us.gcr.io/broad-dsde-methods/broad-gatk-snapshots/gatk-remote-builds:<USERNAME>-<GITHUB_TAG>-<GIT_HASH>`
+This will attempt to build the docker remotely using google cloud build and tag it with the following this format: `us.gcr.io/broad-dsde-methods/broad-gatk-snapshots/gatk-remote-builds:<USERNAME>-<GITHUB_TAG>-<GIT_HASH>`
 
 From this directory, run:
 
@@ -65,7 +65,7 @@ sudo bash build_docker_remote.sh -e ${GITHUB_TAG} -d ${STAGING_DIR}
 
 #### Create GATK4 docker image and upload into GCR (with a custom gcr repo tag)
 
-From this directory, run:
+This will attempt to build the docker remotely using google cloud build. From this directory, run:
 
 ```bash
 # REPLACE VALUE OF GITHUB_TAG WITH DESIRED VERSION