From 55f55d4ed81520ed74dbc3b23985433513d32ef0 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Fri, 12 Jan 2024 13:56:46 -0500
Subject: [PATCH 01/15] Adopt new Google Cloud SDK, `gcloud storage`

---
 .../pipelines/common/action/ActionUtils.scala |  2 +-
 .../v2beta/src/main/resources/gcs_transfer.sh | 19 +++++++++++--------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
index b05971a13ad..4aad9364fdb 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
@@ -32,7 +32,7 @@ object ActionUtils {
     * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
     */
   val CloudSdkImage: String =
-    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:354.0.0-alpine")
+    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:459.0.0-alpine")
 
   /*
    * At the moment, cloud-sdk (584MB for 354.0.0-alpine) and stedolan/jq (182MB) decompressed ~= 0.8 GB
diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index 47dee2d67df..8f0080fb92f 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -34,9 +34,11 @@ private::delocalize_file() {
   cloud_parent=$(dirname "$cloud")"/"
 
   if [[ -f "$container" && -n "$content" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -h "Content-Type: $content" -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    # TODO weird error caused by headers
+    # https://broadinstitute.slack.com/archives/C3GMDRWCS/p1705084799477529
+    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} --additional-headers=Content-Type="$content" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -f "$container" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -e "$container" ]]; then
     echo "File output '$container' exists but is not a file"
     exit 1
@@ -163,15 +165,16 @@ localize_files() {
   fi
 
   # We need to determine requester pays status of the first file attempting at most `max_attempts` times.
-  NO_REQUESTER_PAYS_COMMAND="mkdir -p '$container_parent' && rm -f "$HOME/.config/gcloud/gce" && gsutil -o 'GSUtil:parallel_thread_count=1' -o 'GSUtil:sliced_object_download_max_components=${num_cpus}' cp '$first_cloud_file' '$container_parent'"
-  REQUESTER_PAYS_COMMAND="rm -f "$HOME/.config/gcloud/gce" && gsutil -o 'GSUtil:parallel_thread_count=1' -o 'GSUtil:sliced_object_download_max_components=${num_cpus}' -u $project cp '$first_cloud_file' '$container_parent'"
+  NO_REQUESTER_PAYS_COMMAND="mkdir -p '$container_parent' && rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp '$first_cloud_file' '$container_parent'"
+  REQUESTER_PAYS_COMMAND="rm -f "$HOME/.config/gcloud/gce" && gcloud storage --billing-project $project cp '$first_cloud_file' '$container_parent'"
 
   basefile=$(basename "$first_cloud_file")
   private::localize_message "$first_cloud_file" "${container_parent}${basefile}"
   private::determine_requester_pays ${max_attempts}
 
   if [[ ${USE_REQUESTER_PAYS} = true ]]; then
-    rpflag="-u $project"
+    # https://cloud.google.com/storage/docs/using-requester-pays#using
+    rpflag="--billing-project $project"
   else
     rpflag=""
   fi
@@ -192,7 +195,7 @@ localize_files() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
       # parallel transfer the remaining files
       rm -f "$HOME/.config/gcloud/gce"
-      if cat files_to_localize.txt | gsutil -o "GSUtil:parallel_thread_count=1" -o "GSUtil:sliced_object_download_max_components=${num_cpus}" -m ${rpflag} cp -I "$container_parent"; then
+      if cat files_to_localize.txt | gcloud storage ${rpflag} cp --read-paths-from-stdin "$container_parent"; then
         break
       else
         attempt=$((attempt + 1))
@@ -247,7 +250,7 @@ localize_directories() {
   private::determine_requester_pays ${max_attempts}
 
   if [[ ${USE_REQUESTER_PAYS} = true ]]; then
-    rpflag="-u $project"
+    rpflag="--billing-project $project"
   else
     rpflag=""
   fi
@@ -311,7 +314,7 @@ delocalize() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
 
       if [[ ${use_requester_pays} = true ]]; then
-        rpflag="-u ${project}"
+        rpflag="--billing-project ${project}"
       else
         rpflag=""
       fi

From 8dd70afe20afa441784cbc6170f31fe1ce0717b3 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Fri, 12 Jan 2024 16:40:59 -0500
Subject: [PATCH 02/15] Fix content type

---
 .../pipelines/v2beta/src/main/resources/gcs_transfer.sh       | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index 8f0080fb92f..add44cad3af 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -34,9 +34,7 @@ private::delocalize_file() {
   cloud_parent=$(dirname "$cloud")"/"
 
   if [[ -f "$container" && -n "$content" ]]; then
-    # TODO weird error caused by headers
-    # https://broadinstitute.slack.com/archives/C3GMDRWCS/p1705084799477529
-    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} --additional-headers=Content-Type="$content" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} --content-type="$content" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -f "$container" ]]; then
     rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -e "$container" ]]; then

From 1dc58d7dacdcb5227990f8eaccdb38c6960b84a0 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Fri, 12 Jan 2024 17:40:45 -0500
Subject: [PATCH 03/15] Fix parameter order

---
 .../pipelines/v2beta/src/main/resources/gcs_transfer.sh   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index add44cad3af..bf7e38b9a8c 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -34,9 +34,9 @@ private::delocalize_file() {
   cloud_parent=$(dirname "$cloud")"/"
 
   if [[ -f "$container" && -n "$content" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} --content-type="$content" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp ${rpflag} --content-type="$content" "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -f "$container" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${rpflag} cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp ${rpflag} "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -e "$container" ]]; then
     echo "File output '$container' exists but is not a file"
     exit 1
@@ -164,7 +164,7 @@ localize_files() {
 
   # We need to determine requester pays status of the first file attempting at most `max_attempts` times.
   NO_REQUESTER_PAYS_COMMAND="mkdir -p '$container_parent' && rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp '$first_cloud_file' '$container_parent'"
-  REQUESTER_PAYS_COMMAND="rm -f "$HOME/.config/gcloud/gce" && gcloud storage --billing-project $project cp '$first_cloud_file' '$container_parent'"
+  REQUESTER_PAYS_COMMAND="rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp --billing-project $project '$first_cloud_file' '$container_parent'"
 
   basefile=$(basename "$first_cloud_file")
   private::localize_message "$first_cloud_file" "${container_parent}${basefile}"
@@ -193,7 +193,7 @@ localize_files() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
       # parallel transfer the remaining files
       rm -f "$HOME/.config/gcloud/gce"
-      if cat files_to_localize.txt | gcloud storage ${rpflag} cp --read-paths-from-stdin "$container_parent"; then
+      if cat files_to_localize.txt | gcloud storage cp ${rpflag} --read-paths-from-stdin "$container_parent"; then
         break
       else
         attempt=$((attempt + 1))

From cec87c1526988832c66f86ecdcb59070dce5bc5e Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Mon, 22 Jan 2024 15:50:12 -0500
Subject: [PATCH 04/15] Download only

---
 .../v2beta/src/main/resources/gcs_transfer.sh      | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index bf7e38b9a8c..ec41ba4bab9 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -34,9 +34,9 @@ private::delocalize_file() {
   cloud_parent=$(dirname "$cloud")"/"
 
   if [[ -f "$container" && -n "$content" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp ${rpflag} --content-type="$content" "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -h "Content-Type: $content" -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container"
   elif [[ -f "$container" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gcloud storage cp ${rpflag} "$container" "$cloud_parent" > "$gsutil_log" 2>&1
+    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container"
   elif [[ -e "$container" ]]; then
     echo "File output '$container' exists but is not a file"
     exit 1
@@ -172,9 +172,9 @@ localize_files() {
 
   if [[ ${USE_REQUESTER_PAYS} = true ]]; then
     # https://cloud.google.com/storage/docs/using-requester-pays#using
-    rpflag="--billing-project $project"
+    gcloud_storage_rpflag="--billing-project $project"
   else
-    rpflag=""
+    gcloud_storage_rpflag=""
   fi
 
 
@@ -193,7 +193,7 @@ localize_files() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
       # parallel transfer the remaining files
       rm -f "$HOME/.config/gcloud/gce"
-      if cat files_to_localize.txt | gcloud storage cp ${rpflag} --read-paths-from-stdin "$container_parent"; then
+      if cat files_to_localize.txt | gcloud storage cp ${gcloud_storage_rpflag} --read-paths-from-stdin "$container_parent"; then
         break
       else
         attempt=$((attempt + 1))
@@ -248,7 +248,7 @@ localize_directories() {
   private::determine_requester_pays ${max_attempts}
 
   if [[ ${USE_REQUESTER_PAYS} = true ]]; then
-    rpflag="--billing-project $project"
+    rpflag="-u $project"
   else
     rpflag=""
   fi
@@ -312,7 +312,7 @@ delocalize() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
 
       if [[ ${use_requester_pays} = true ]]; then
-        rpflag="--billing-project ${project}"
+        rpflag="-u $project"
       else
         rpflag=""
       fi

From bc5d2ae7c8091ea3be3f6c5d6811f103ec339add Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Mon, 22 Jan 2024 15:51:22 -0500
Subject: [PATCH 05/15] Fix

---
 .../google/pipelines/v2beta/src/main/resources/gcs_transfer.sh  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index ec41ba4bab9..5856e8792a1 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -312,7 +312,7 @@ delocalize() {
     while [[ ${attempt} -le ${max_attempts} ]]; do
 
       if [[ ${use_requester_pays} = true ]]; then
-        rpflag="-u $project"
+        rpflag="-u ${project}"
       else
         rpflag=""
       fi

From 50371692e9ca6d99a717afc3b3ebfc6002b7c5c5 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Mon, 22 Jan 2024 15:52:33 -0500
Subject: [PATCH 06/15] Fix

---
 .../pipelines/v2beta/src/main/resources/gcs_transfer.sh       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index 5856e8792a1..5ec0922f928 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -34,9 +34,9 @@ private::delocalize_file() {
   cloud_parent=$(dirname "$cloud")"/"
 
   if [[ -f "$container" && -n "$content" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -h "Content-Type: $content" -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container"
+    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -h "Content-Type: $content" -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -f "$container" ]]; then
-    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container"
+    rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m -o GSUtil:parallel_composite_upload_threshold="$parallel_composite_upload_threshold" cp "$container" "$cloud_parent" > "$gsutil_log" 2>&1
   elif [[ -e "$container" ]]; then
     echo "File output '$container' exists but is not a file"
     exit 1

From edb97067dbe7f9949cba25f5b0b81277c363cbb5 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Mon, 22 Jan 2024 16:37:10 -0500
Subject: [PATCH 07/15] Directory localization

---
 .../v2beta/src/main/resources/gcs_transfer.sh      | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
index 5ec0922f928..f729f46bdc0 100755
--- a/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
+++ b/supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh
@@ -210,13 +210,13 @@ private::localize_directory() {
   local cloud="$1"
   local container="$2"
   local max_attempts="$3"
-  local rpflag="$4"
+  local gcloud_storage_rpflag="$4"
 
   local attempt=1
   private::localize_message "$cloud" "$container"
   while [[ ${attempt} -lt ${max_attempts} ]]; do
-    # Do not quote rpflag, when that is set it will be -u project which should be two distinct arguments.
-    if mkdir -p "${container}" && rm -f "$HOME/.config/gcloud/gce" && gsutil ${rpflag} -m rsync -r "${cloud}" "${container}" > /dev/null 2>&1; then
+    # Do not quote gcloud_storage_rpflag, when that is set it will be `--billing-project project` which should be two distinct arguments.
+    if mkdir -p "${container}" && rm -f "$HOME/.config/gcloud/gce" && gcloud storage ${gcloud_storage_rpflag} rsync -r "${cloud}" "${container}" > /dev/null 2>&1; then
       break
     else
       attempt=$(($attempt + 1))
@@ -243,21 +243,21 @@ localize_directories() {
 
   BASE_COMMAND="private::localize_directory '${cloud_directory}' '${container_directory}' '${max_attempts}'"
   NO_REQUESTER_PAYS_COMMAND="${BASE_COMMAND} ''"
-  REQUESTER_PAYS_COMMAND="${BASE_COMMAND} '-u $project'"
+  REQUESTER_PAYS_COMMAND="${BASE_COMMAND} '--billing-project $project'"
 
   private::determine_requester_pays ${max_attempts}
 
   if [[ ${USE_REQUESTER_PAYS} = true ]]; then
-    rpflag="-u $project"
+    gcloud_storage_rpflag="--billing-project $project"
   else
-    rpflag=""
+    gcloud_storage_rpflag=""
   fi
 
   while [[ $# -gt 0 ]]; do
     cloud_directory="$1"
     container_directory="$2"
     shift 2
-    private::localize_directory "$cloud_directory" "$container_directory" "$max_attempts" "$rpflag"
+    private::localize_directory "$cloud_directory" "$container_directory" "$max_attempts" "$gcloud_storage_rpflag"
   done
 }
 

From 9e38e13e48bf12804a4d9c2d35f1d40200ef69c6 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Tue, 23 Jan 2024 22:02:04 -0500
Subject: [PATCH 08/15] Google Cloud SDK 460.0.0

---
 .../backend/google/pipelines/common/action/ActionUtils.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
index 4aad9364fdb..8f60a999a32 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
@@ -32,7 +32,7 @@ object ActionUtils {
     * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
     */
   val CloudSdkImage: String =
-    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:459.0.0-alpine")
+    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:460.0.0-alpine")
 
   /*
    * At the moment, cloud-sdk (584MB for 354.0.0-alpine) and stedolan/jq (182MB) decompressed ~= 0.8 GB

From 9abc4d5690b6f723bdb225163b2f315d8f162f82 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Tue, 23 Jan 2024 22:08:23 -0500
Subject: [PATCH 09/15] Use self-hosted SDK [PROD-918]

---
 .../backend/google/pipelines/common/action/ActionUtils.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
index 8f60a999a32..7f63674c936 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
@@ -32,7 +32,7 @@ object ActionUtils {
     * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
     */
   val CloudSdkImage: String =
-    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:460.0.0-alpine")
+    config.getOrElse("cloud-sdk-image-url", "us.gcr.io/broad-dsp-gcr-public/cloudsdktool/cloud-sdk:460.0.0-alpine")
 
   /*
    * At the moment, cloud-sdk (584MB for 354.0.0-alpine) and stedolan/jq (182MB) decompressed ~= 0.8 GB

From 37b3534843402f993f39bd3ca20355fa2a661a3f Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Tue, 23 Jan 2024 22:47:24 -0500
Subject: [PATCH 10/15] Breadcrumbs

---
 .../google/batch/runnable/RunnableUtils.scala  | 18 +++---------------
 .../pipelines/common/action/ActionUtils.scala  | 17 ++++++-----------
 2 files changed, 9 insertions(+), 26 deletions(-)

diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala
index f60baed0608..d20248dc79b 100644
--- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala
+++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala
@@ -16,30 +16,18 @@ object RunnableUtils {
   /** Port mappings for the ssh container. */
   val sshPortMappings = Map("22" -> Int.box(22))
 
-  /*
-   * At the moment, cloud-sdk (924MB for 276.0.0-slim) and stedolan/jq (182MB) decompressed ~= 1.1 GB
-   */
-  val cromwellImagesSizeRoundedUpInGB = 1
-
   private val config = ConfigFactory.load().getConfig("google")
 
   /**
     * An image with the Google Cloud SDK installed.
     * http://gcr.io/google.com/cloudsdktool/cloud-sdk
     *
-    * FYI additional older versions are available on DockerHub at:
-    * https://hub.docker.com/r/google/cloud-sdk
+    * As of PROD-918 in January 2024, we are re-hosting ourselves due to quota issues pulling the official image.
     *
-    * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
+    * Also update `cromwell.backend.google.pipelines.common.action.ActionUtils`
     */
   val CloudSdkImage: String =
-    // config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:354.0.0-alpine")
-    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:434.0.0-alpine")
-  /*
-   * At the moment, cloud-sdk (584MB for 354.0.0-alpine) and stedolan/jq (182MB) decompressed ~= 0.8 GB
-   */
-  val CromwellImagesSizeRoundedUpInGB: Int =
-    config.getOrElse("cloud-sdk-image-size-gb", 1)
+    config.getOrElse("cloud-sdk-image-url", "us.gcr.io/broad-dsp-gcr-public/cloudsdktool/cloud-sdk:460.0.0-alpine")
 
   /** Quotes a string such that it's compatible as a string argument in the shell. */
   def shellEscaped(any: Any): String = {
diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
index 7f63674c936..82c8761eba0 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/action/ActionUtils.scala
@@ -15,30 +15,25 @@ object ActionUtils {
   /** Port mappings for the ssh container. */
   val sshPortMappings = Map("22" -> Int.box(22))
 
-  /*
-   * At the moment, cloud-sdk (924MB for 276.0.0-slim) and stedolan/jq (182MB) decompressed ~= 1.1 GB
-   */
-  val cromwellImagesSizeRoundedUpInGB = 1
-
   private val config = ConfigFactory.load().getConfig("google")
 
   /**
     * An image with the Google Cloud SDK installed.
     * http://gcr.io/google.com/cloudsdktool/cloud-sdk
     *
-    * FYI additional older versions are available on DockerHub at:
-    * https://hub.docker.com/r/google/cloud-sdk
+    * As of PROD-918 in January 2024, we are re-hosting ourselves due to quota issues pulling the official image.
+    *
+    * When updating this value, also consider updating the `cromwellImagesSizeRoundedUpInGB` below.
     *
-    * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
+    * Also update `cromwell.backend.google.batch.runnable.RunnableUtils`
     */
   val CloudSdkImage: String =
     config.getOrElse("cloud-sdk-image-url", "us.gcr.io/broad-dsp-gcr-public/cloudsdktool/cloud-sdk:460.0.0-alpine")
 
   /*
-   * At the moment, cloud-sdk (584MB for 354.0.0-alpine) and stedolan/jq (182MB) decompressed ~= 0.8 GB
+   * At the moment, cloud-sdk (960MB for 460.0.0-alpine) ~= 1.0 GB
    */
-  val CromwellImagesSizeRoundedUpInGB: Int =
-    config.getOrElse("cloud-sdk-image-size-gb", 1)
+  val cromwellImagesSizeRoundedUpInGB: Int = config.getOrElse("cloud-sdk-image-size-gb", 1)
 
   /** Quotes a string such that it's compatible as a string argument in the shell. */
   def shellEscaped(any: Any): String = {

From 39a8567ba2adeef64cafcec8159a88594bb72b33 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Tue, 23 Jan 2024 23:19:31 -0500
Subject: [PATCH 11/15] Changelog

---
 CHANGELOG.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a581852c02e..3791624ede0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Cromwell Change Log
 
+## 87 Release Notes
+
+### Replacement of `gsutil` with `gcloud storage`
+In this release, all _localization_ functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2].
+
+In a future release, _delocalization_ will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). 
+
+[0] Tested with Intel Ice Lake CPU platform, 16 vCPU, 32 GB RAM, 2500 GB SSD
+
+[1] [Throughput scales with vCPU count](https://cloud.google.com/compute/docs/disks/performance#n2_vms) with a plateau at 16 vCPUs.
+
+[2] [Throughput scales with disk size.](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) To achieve maximum performance: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB.
+
 ## 86 Release Notes
 
 ### GCP Batch

From a88c8dfc0cbc7de24b9db35dc168795ddd9bcfa1 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Tue, 23 Jan 2024 23:28:49 -0500
Subject: [PATCH 12/15] Formatting

---
 CHANGELOG.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3791624ede0..72fc81f378e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,9 +3,10 @@
 ## 87 Release Notes
 
 ### Replacement of `gsutil` with `gcloud storage`
-In this release, all _localization_ functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2].
 
-In a future release, _delocalization_ will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). 
+In this release, all **localization** functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2].
+
+In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). 
 
 [0] Tested with Intel Ice Lake CPU platform, 16 vCPU, 32 GB RAM, 2500 GB SSD
 

From 476e62c41d022bf21fcd0c68853f4ca3c284c2f1 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Wed, 24 Jan 2024 10:05:05 -0500
Subject: [PATCH 13/15] Better release notes

---
 CHANGELOG.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72fc81f378e..166ba621543 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,12 @@
 
 In this release, all **localization** functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2].
 
-In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). 
+In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. Delocalized composite objects will no longer have an md5 checksum in the bucket; refer to the matrix below [3]. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues).
+
+| Delocalization Strategy | Performance   | crc32c | md5 |
+|-------------------------|---------------|--------|-----|
+| Classic                 | Baseline/slow | ✅     | ✅  |
+| Parallel Composite      | Fast          | ✅     | ❌  |
 
 [0] Tested with Intel Ice Lake CPU platform, 16 vCPU, 32 GB RAM, 2500 GB SSD
 
@@ -14,6 +19,8 @@ In a future release, **delocalization** will also migrate to `gcloud storage`. A
 
 [2] [Throughput scales with disk size.](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) To achieve maximum performance: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB.
 
+[3] Cromwell itself uses crc32c hashes for call caching and is not affected
+
 ## 86 Release Notes
 
 ### GCP Batch

From 2cec1451477799469c47ed2230ebdfec13897b20 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Wed, 24 Jan 2024 13:18:18 -0500
Subject: [PATCH 14/15] Changelog perfectionism

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 166ba621543..798770edd71 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,7 @@ In a future release, **delocalization** will also migrate to `gcloud storage`. A
 
 [1] [Throughput scales with vCPU count](https://cloud.google.com/compute/docs/disks/performance#n2_vms) with a plateau at 16 vCPUs.
 
-[2] [Throughput scales with disk size.](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) To achieve maximum performance: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB.
+[2] [Throughput scales with disk size and type](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) with at a plateau at 2.5 TB SSD. Worked example: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB.
 
 [3] Cromwell itself uses crc32c hashes for call caching and is not affected
 

From c7b7615bbf2da4e60c40381fbc235d67e954ab53 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Wed, 24 Jan 2024 13:21:28 -0500
Subject: [PATCH 15/15] Changelog perfectionism

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 798770edd71..9f4d9c30560 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@
 
 In this release, all **localization** functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2].
 
-In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. Delocalized composite objects will no longer have an md5 checksum in the bucket; refer to the matrix below [3]. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues).
+In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. Delocalized composite objects will no longer have an md5 checksum in their metadata; refer to the matrix below [3]. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues).
 
 | Delocalization Strategy | Performance   | crc32c | md5 |
 |-------------------------|---------------|--------|-----|