From 29b374f6d37e8cf6d1bfc259889f518660190ab8 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 20 Jul 2023 10:57:09 -0400
Subject: [PATCH 01/36] add json dependency

---
 .../scala/drs/localizer/DrsLocalizerMain.scala   | 16 +++++++++++++++-
 project/Dependencies.scala                       |  1 +
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 1858f395024..04f54d6699e 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -11,12 +11,15 @@ import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
 import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
-
 import java.io.File
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
+import spray.json._
+import DefaultJsonProtocol._
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
 
 object DrsLocalizerMain extends IOApp with StrictLogging {
 
@@ -76,6 +79,17 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     new DrsLocalizerMain(drsObject, containerPath, drsCredentials, commandLineArguments.googleRequesterPaysProject).
       resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode)
   }
+
+  private def localizeManifest(csvManifestPath : String) = {
+    // The DRS localizer is provided a manifest in CSV format.
+    // The getm downloader expects a JSON manifest.
+    // Here, we convert the CSV into a JSON and save it to a new file.
+    val csvManifestFile = new File(csvManifestPath)
+    val csvParser = CSVParser.parse(csvManifestFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
+    val scalaMap = csvParser.asScala.map(record => (record.get(0), record.get(1))).toMap
+    val jsonString = scalaMap.toJson.toString
+    Files.write(Paths.get("manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
+  }
 }
 
 class DrsLocalizerMain(drsUrl: String,
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 1a7681da1e3..122d224a66d 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -644,6 +644,7 @@ object Dependencies {
     "com.softwaremill.sttp" %% "circe" % sttpV,
     "com.github.scopt" %% "scopt" % scoptV,
     "org.apache.commons" % "commons-csv" % commonsCsvV,
+    "org.gnieh" %% "diffson-spray-json" % diffsonSprayJsonV
   ) ++ circeDependencies ++ catsDependencies ++ slf4jBindingDependencies ++ languageFactoryDependencies ++ azureDependencies
 
   val allProjectDependencies: List[ModuleID] =

From ddf0328f9b8926cc8be16b7707a4cdec88887892 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 20 Jul 2023 16:56:53 -0400
Subject: [PATCH 02/36] make it scala

---
 .../drs/localizer/DrsLocalizerMain.scala      | 35 +++++++++++++------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 04f54d6699e..f9b55b50a61 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -11,15 +11,18 @@ import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
 import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
-import java.io.File
+
+import java.io.{File, IOException}
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import spray.json._
 import DefaultJsonProtocol._
-import java.nio.file.{Paths, Files}
+
+import java.nio.file.{Files, Paths}
 import java.nio.charset.StandardCharsets
+import scala.util.{Failure, Success, Try}
 
 object DrsLocalizerMain extends IOApp with StrictLogging {
 
@@ -80,16 +83,26 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode)
   }
 
-  private def localizeManifest(csvManifestPath : String) = {
-    // The DRS localizer is provided a manifest in CSV format.
-    // The getm downloader expects a JSON manifest.
-    // Here, we convert the CSV into a JSON and save it to a new file.
-    val csvManifestFile = new File(csvManifestPath)
-    val csvParser = CSVParser.parse(csvManifestFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
-    val scalaMap = csvParser.asScala.map(record => (record.get(0), record.get(1))).toMap
-    val jsonString = scalaMap.toJson.toString
-    Files.write(Paths.get("manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
+
+  // The DRS localizer is provided a manifest in CSV format.
+  private def loadCSVAsJson(csvManifestPath : String) : Try[JsValue] = {
+    val scalaMap : Try[Map[String, String]] = for {
+      openFile <- Try(new File(csvManifestPath))
+      csvParser <- Try(CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT))
+      map <- Try(csvParser.getRecords.asScala.map(record => (record.get(0), record.get(1))).toMap)
+      if !map.isEmpty
+    } yield map
+    Try(scalaMap.get.toJson)
   }
+
+  private def writeJsonManifest(destinationPath : String, jsonData : JsValue): Try[String] = {
+    Try(Files.write(Paths.get(destinationPath), jsonData.toString().getBytes(StandardCharsets.UTF_8)).toString)
+  }
+  //getm 
+  private def generateJsonPathForManifest(csvPath : String) : String = {
+    csvPath.substring(0, csvPath.lastIndexOf('.')) + "_manifest.json"
+  }
+
 }
 
 class DrsLocalizerMain(drsUrl: String,

From 7d6cd8b96462c20204036e5d574e6470b637dfbf Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 21 Jul 2023 16:21:34 -0400
Subject: [PATCH 03/36] lots of scaffolding

---
 .../drs/localizer/DrsLocalizerMain.scala      | 42 +++++++++++++++----
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index f9b55b50a61..d64662b2911 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -83,26 +83,44 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode)
   }
 
-
-  // The DRS localizer is provided a manifest in CSV format.
-  private def loadCSVAsJson(csvManifestPath : String) : Try[JsValue] = {
-    val scalaMap : Try[Map[String, String]] = for {
+  // Overall plan for bulk DRS downloads:
+  // 1) Open the CSV provided to Cromwell Drs Localizer as a map of DRS URI -> Destination Filepath
+  // 2) Resolve each DRS URI in the map. This will yield two maps, because a drs URI can resolve to either
+  // a gs:// URI or an https:// uri, each of which require different downloaders.
+  // 3a) Invoke the GcsUriDownloader (and its underlying gsutil) to download all gs:// URIs in parallel.
+  // 3b) Invoke the AccessUrlDownloader (and its underlying getm) to download all https:// URIs in parallel.
+  //     - getm requires a --manifest argument, which is a .json file that is a map of https:// URI to destination filepath
+  // 4) Report success or failure as granularly as is reasonably possible.
+  private def loadCSVAsMap(csvManifestPath : String) : Try[Map[String, String]] = {
+    for {
       openFile <- Try(new File(csvManifestPath))
       csvParser <- Try(CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT))
       map <- Try(csvParser.getRecords.asScala.map(record => (record.get(0), record.get(1))).toMap)
-      if !map.isEmpty
+      if !map.isEmpty //An empty map should be considered an error - we're expecting at least one k/v pair in the manifest.
     } yield map
-    Try(scalaMap.get.toJson)
   }
 
+  private def resolveURIs(drsUrlToDownloadLocationMap : Map[String,String]) : Try[Map[String, String]] = {
+    val accessUrlToDownloadLocationMap = drsUrlToDownloadLocationMap.map((drsUrl, downloadLocation) => resolve)
+  }
+
+  //Write string to file. Returns the filepath that the string was written to.
   private def writeJsonManifest(destinationPath : String, jsonData : JsValue): Try[String] = {
     Try(Files.write(Paths.get(destinationPath), jsonData.toString().getBytes(StandardCharsets.UTF_8)).toString)
   }
-  //getm 
+  // We're going to write a JSON version of the CSV manifest we're provided.
+  // Save the new manifest to wherever the CSV is saved, but replace .csv with _manifest.json.
   private def generateJsonPathForManifest(csvPath : String) : String = {
     csvPath.substring(0, csvPath.lastIndexOf('.')) + "_manifest.json"
   }
 
+  // The getm tool can read in a JSON manifest and download all files in parallel.
+  // Here, we invoke it with the provided manifest file.
+  // The manifest file should have 1 or more key value pairs, where each key is a drs URI
+  // and each value is a destination path for where that file should be localized to.
+  private def downloadInParallel(jsonManifestPath : String) : IO[ExitCode] = {
+
+  }
 }
 
 class DrsLocalizerMain(drsUrl: String,
@@ -190,4 +208,14 @@ class DrsLocalizerMain(drsUrl: String,
       }
     } yield downloader
   }
+
+  //TODO: use this fn to resolve drs URIs, split the responses into gs:// and https:// groups, and invoke new downloaders for each of those groups.
+  private[localizer] def resolveDrsUrl(drsUrl: String): IO[DrsResolverResponse] = {
+    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+    for {
+      resolver <- getDrsPathResolver
+      drsResolverResponse <- resolver.resolveDrs(drsUrl, fields)
+    } yield drsResolverResponse
+  }
+
 }

From be95d22121578783efb33a577a4b2ceee2db69db Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 3 Aug 2023 16:42:03 -0400
Subject: [PATCH 04/36] forward progress

---
 .../drs/localizer/DrsLocalizerMain.scala      |  53 +-
 ...w.194f43e1-319b-402d-9c1d-ff704a1db2a2.log | 585 +++++++++++++++
 ...w.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log | 670 ++++++++++++++++++
 .../TesAsyncBackendJobExecutionActor.scala    |   5 +-
 4 files changed, 1308 insertions(+), 5 deletions(-)
 create mode 100644 https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
 create mode 100644 https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index d64662b2911..c90fdac305c 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -19,6 +19,8 @@ import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import spray.json._
 import DefaultJsonProtocol._
+import drs.localizer.DrsLocalizerMain.{downloadInParallel, generateJsonPathForManifest, loadCSVAsMap, writeJsonManifest}
+import drs.localizer.URIType.{GCS, URIType}
 
 import java.nio.file.{Files, Paths}
 import java.nio.charset.StandardCharsets
@@ -123,6 +125,11 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
   }
 }
 
+object URIType extends Enumeration {
+  type URIType = Value
+  val GCS, HTTPS = Value
+}
+
 class DrsLocalizerMain(drsUrl: String,
                        downloadLoc: String,
                        drsCredentials: DrsCredentials,
@@ -210,12 +217,52 @@ class DrsLocalizerMain(drsUrl: String,
   }
 
   //TODO: use this fn to resolve drs URIs, split the responses into gs:// and https:// groups, and invoke new downloaders for each of those groups.
-  private[localizer] def resolveDrsUrl(drsUrl: String): IO[DrsResolverResponse] = {
+  private[localizer] def resolveDrsUrl(resolverObject: IO[DrsLocalizerDrsPathResolver], drsUrlToResolve: String): IO[DrsResolverResponse] = {
     val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
     for {
-      resolver <- getDrsPathResolver
-      drsResolverResponse <- resolver.resolveDrs(drsUrl, fields)
+      resolver <- resolverObject
+      drsResolverResponse <- resolver.resolveDrs(drsUrlToResolve, fields)
     } yield drsResolverResponse
   }
 
+  private[localizer] def toUriType(drsResponse: DrsResolverResponse) : Try[URIType] = {
+    (drsResponse.accessUrl, drsResponse.gsUri) match {
+      case (Some(accessUrl), _) =>
+        Try(URIType.HTTPS)
+      case (_, Some(gcsPath)) =>
+        Try(URIType.GCS)
+      case _ =>
+        Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
+    }
+  }
+
+  private[localizer] def bulkResolve(): Unit = {
+    // Load path resolver once from config
+    val resolver = getDrsPathResolver
+
+    // resolve all the URIs once.
+    val resolvedUris : Try[Map[DrsResolverResponse, String]] = for {
+      drsUriToDownloadLocationMap : Map[String, String] <- loadCSVAsMap("path/to/manifest")
+      resolvedUriToDownloadLocationMap : Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1), value._2))
+    } yield resolvedUriToDownloadLocationMap
+
+    // find the GCS ones and download them
+    for {
+      drsResponses <- resolvedUris
+      gcsUris <- drsResponses.filter(drsResponse => toUriType(drsResponse) == URIType.GCS)
+    }
+
+      //find the HTTPS ones and download them
+    for {
+      drsResponses <- resolvedUris
+      httpsUris = drsResponses.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
+      uriToDownloadLocation = httpsUris.map(drsResponseToDownloadLoc => (drsResponseToDownloadLoc._1.accessUrl.get.toString, drsResponseToDownloadLoc._2))
+      jsonString <- uriToDownloadLocation.toJson
+      pathOfJsonFile <- writeJsonManifest(generateJsonPathForManifest("path/to/manifest"), jsonString)
+      downloadSuccess <- downloadInParallel(pathOfJsonFile)
+    }
+
+  }
+
 }
+
diff --git a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
new file mode 100644
index 00000000000..6fc9392fd95
--- /dev/null
+++ b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
@@ -0,0 +1,585 @@
+2023-07-27 11:43:30,007 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Parsing workflow as WDL draft-2
+2023-07-27 11:43:30,016 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Call-to-Backend assignments: wf_hello.hello -> TES
+2023-07-27 11:43:30,407 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:45:40,558 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Parsing workflow as WDL draft-2
+2023-07-27 11:45:40,567 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Call-to-Backend assignments: wf_hello.hello -> TES
+2023-07-27 11:45:41,848 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:46:42,262 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 2. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:47:42,795 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 3. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:48:43,290 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 4. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:49:43,688 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 5. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:50:44,159 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 6. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
diff --git a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log
new file mode 100644
index 00000000000..52d8225d72b
--- /dev/null
+++ b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log
@@ -0,0 +1,670 @@
+2023-07-27 11:41:05,061 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
+2023-07-27 11:41:05,559 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
+2023-07-27 11:41:07,007 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:42:56,106 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
+2023-07-27 11:42:56,535 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
+2023-07-27 11:42:57,853 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:45:39,659 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
+2023-07-27 11:45:40,103 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
+2023-07-27 11:45:41,493 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:46:41,908 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 2. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:47:42,407 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 3. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:48:42,874 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 4. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:49:43,277 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 5. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
+2023-07-27 11:50:43,751 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 6. Will retry up to 16 times. Next retry is in 1 minute
+cromwell.core.CromwellAggregatedException: Initialization Failure:
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
+	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
+	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
+	at akka.actor.FSM.processEvent(FSM.scala:707)
+	at akka.actor.FSM.processEvent$(FSM.scala:704)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
+	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
+	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
+	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
+	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
+	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
+	at akka.actor.Actor.aroundReceive(Actor.scala:539)
+	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
+	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.Timers.aroundReceive(Timers.scala:55)
+	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
+	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
+	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
+	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
+	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
+	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
+	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
+	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
+	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
+	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
+	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
+	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
+<title>401 Unauthorized</title>
+</head><body>
+<h1>Unauthorized</h1>
+<p>This server could not verify that you
+are authorized to access the document
+requested.  Either you supplied the wrong
+credentials (e.g., bad password), or your
+browser doesn't understand how to supply
+the credentials required.</p>
+</body></html>
+
+		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
+		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
+		at scala.util.Try$.apply(Try.scala:210)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
+		at scala.util.Success.flatMap(Try.scala:258)
+		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
+		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
+		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
+		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
+		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
+		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
+		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
+		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
+		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
+		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
+		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
+		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
+		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
+		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
+		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
+		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
+		... 4 common frames omitted
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
index 7ff507e63f2..e46212817c4 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -161,8 +161,9 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
         jobShell,
         mode)
     })
-
-    tesTask.map(TesTask.makeTask)
+    val testask = tesTask.map(TesTask.makeTask)
+    jobLogger.info(testask.toString)
+    testask
   }
 
   def writeScriptFile(): Future[Unit] = {

From 251972713b9adb66d13e7881811e120f4ca517f7 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 4 Aug 2023 12:41:27 -0400
Subject: [PATCH 05/36] stash

---
 .../main/scala/drs/localizer/DrsLocalizerMain.scala | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index c90fdac305c..43fc8f832d2 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -102,10 +102,6 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     } yield map
   }
 
-  private def resolveURIs(drsUrlToDownloadLocationMap : Map[String,String]) : Try[Map[String, String]] = {
-    val accessUrlToDownloadLocationMap = drsUrlToDownloadLocationMap.map((drsUrl, downloadLocation) => resolve)
-  }
-
   //Write string to file. Returns the filepath that the string was written to.
   private def writeJsonManifest(destinationPath : String, jsonData : JsValue): Try[String] = {
     Try(Files.write(Paths.get(destinationPath), jsonData.toString().getBytes(StandardCharsets.UTF_8)).toString)
@@ -247,20 +243,15 @@ class DrsLocalizerMain(drsUrl: String,
     } yield resolvedUriToDownloadLocationMap
 
     // find the GCS ones and download them
-    for {
-      drsResponses <- resolvedUris
-      gcsUris <- drsResponses.filter(drsResponse => toUriType(drsResponse) == URIType.GCS)
-    }
-
       //find the HTTPS ones and download them
-    for {
+    val exitCode: IO[ExitCode] = for {
       drsResponses <- resolvedUris
       httpsUris = drsResponses.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
       uriToDownloadLocation = httpsUris.map(drsResponseToDownloadLoc => (drsResponseToDownloadLoc._1.accessUrl.get.toString, drsResponseToDownloadLoc._2))
       jsonString <- uriToDownloadLocation.toJson
       pathOfJsonFile <- writeJsonManifest(generateJsonPathForManifest("path/to/manifest"), jsonString)
       downloadSuccess <- downloadInParallel(pathOfJsonFile)
-    }
+    } yield downloadSuccess
 
   }
 

From 81f19a5e8b46e557ccf1dd9f1a99666ea3354c97 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 4 Aug 2023 16:29:17 -0400
Subject: [PATCH 06/36] more progress

---
 .../drs/localizer/DrsLocalizerMain.scala      | 132 +++++++++++-------
 .../downloaders/BulkAccessUrlDownloader.scala | 112 +++++++++++++++
 .../downloaders/DownloaderFactory.scala       |   2 +
 3 files changed, 194 insertions(+), 52 deletions(-)
 create mode 100644 cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 43fc8f832d2..9c7754b9944 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -19,7 +19,6 @@ import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import spray.json._
 import DefaultJsonProtocol._
-import drs.localizer.DrsLocalizerMain.{downloadInParallel, generateJsonPathForManifest, loadCSVAsMap, writeJsonManifest}
 import drs.localizer.URIType.{GCS, URIType}
 
 import java.nio.file.{Files, Paths}
@@ -55,6 +54,10 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
 
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
       IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+
+    override def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap: Map[AccessUrl, String], hashes: Hashes): IO[Downloader] = {
+      IO.pure(BulkAccessUrlDownloader(urlToDownloadDestinationMap, hashes))
+    }
   }
 
   private def printUsage: IO[ExitCode] = {
@@ -80,6 +83,82 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
 
+  def getDrsPathResolver(drsCredentials: DrsCredentials): IO[DrsLocalizerDrsPathResolver] = {
+    IO {
+      val drsConfig = DrsConfig.fromEnv(sys.env)
+      logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects")
+      new DrsLocalizerDrsPathResolver(drsConfig, drsCredentials)
+    }
+  }
+
+  private[localizer] def toUriType(drsResponse: DrsResolverResponse): Try[URIType] = {
+    (drsResponse.accessUrl, drsResponse.gsUri) match {
+      case (Some(accessUrl), _) =>
+        Try(URIType.HTTPS)
+      case (_, Some(gcsPath)) =>
+        Try(URIType.GCS)
+      case _ =>
+        Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
+    }
+  }
+  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials, dummy : Boolean): IO[ExitCode] = {
+    commandLineArguments.manifestPath match {
+      case Some(manifestPath) => {
+          val resolvedUris: Try[Map[DrsResolverResponse, String]] = for {
+            resolver: IO[DrsLocalizerDrsPathResolver] <- getDrsPathResolver(drsCredentials)
+            resolvedDrsUris: Try[Map[DrsResolverResponse, String]] <- resolveCSVManifest(manifestPath, resolver)
+            if !resolvedDrsUris.isEmpty
+          } yield resolvedDrsUris
+
+          if(resolvedUris.isFailure) {
+              return IO(ExitCode.Error) //Could not resolve DRS urls. Fail before attempting to download anything
+            }
+
+          val accessUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
+          val googleUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.GCS)
+
+          val bulkAccessResult : IO[ExitCode] = bulkDownloadResolvedAccessUrls(accessUrls, drsCredentials)
+          val googleResult : List[IO[DownloadResult]] = bulkDownloadResolvedGoogleUrls(googleUrls, commandLineArguments.googleRequesterPaysProject)
+
+          //TODO: return an appropriate exit code
+        }
+      case None =>
+        val drsObject = commandLineArguments.drsObject.get
+        val containerPath = commandLineArguments.containerPath.get
+        localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
+    }
+  }
+
+  def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
+    //TODO: invoke new bulk getm downloader
+  }
+
+  def bulkDownloadResolvedGoogleUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], googleRequesterPaysProject : Option[String]) : List[IO[DownloadResult]] = {
+    resolvedUrlToOutputDestination.map(pair => {
+      val drsResponse : DrsResolverResponse = pair._1
+      val containerPath : String = pair._2
+      val serviceAccountJsonOption = drsResponse.googleServiceAccount.map(_.data.spaces2)
+      val downloader = defaultDownloaderFactory.buildGcsUriDownloader(
+        gcsPath = drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
+        serviceAccountJsonOption = serviceAccountJsonOption,
+        downloadLoc = containerPath,
+        requesterPaysProjectOption = googleRequesterPaysProject)
+      downloader.flatMap(_.download)
+    }).toList
+  }
+
+  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: String): IO[DrsResolverResponse] = {
+    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+    resolverObject.resolveDrs(drsUrlToResolve, fields)
+  }
+  // resolve all the URIs once.
+  def resolveCSVManifest(csvManifestPath: String, resolver: DrsLocalizerDrsPathResolver) : Try[Map[DrsResolverResponse, String]] = {
+      for {
+        drsUriToDownloadLocationMap: Map[String, String] <- loadCSVAsMap(csvManifestPath)
+        resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1), value._2))
+      } yield resolvedUriToDownloadLocationMap
+    }
+
   private def localizeFile(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials, drsObject: String, containerPath: String) = {
     new DrsLocalizerMain(drsObject, containerPath, drsCredentials, commandLineArguments.googleRequesterPaysProject).
       resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode)
@@ -112,13 +191,6 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     csvPath.substring(0, csvPath.lastIndexOf('.')) + "_manifest.json"
   }
 
-  // The getm tool can read in a JSON manifest and download all files in parallel.
-  // Here, we invoke it with the provided manifest file.
-  // The manifest file should have 1 or more key value pairs, where each key is a drs URI
-  // and each value is a destination path for where that file should be localized to.
-  private def downloadInParallel(jsonManifestPath : String) : IO[ExitCode] = {
-
-  }
 }
 
 object URIType extends Enumeration {
@@ -211,49 +283,5 @@ class DrsLocalizerMain(drsUrl: String,
       }
     } yield downloader
   }
-
-  //TODO: use this fn to resolve drs URIs, split the responses into gs:// and https:// groups, and invoke new downloaders for each of those groups.
-  private[localizer] def resolveDrsUrl(resolverObject: IO[DrsLocalizerDrsPathResolver], drsUrlToResolve: String): IO[DrsResolverResponse] = {
-    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-    for {
-      resolver <- resolverObject
-      drsResolverResponse <- resolver.resolveDrs(drsUrlToResolve, fields)
-    } yield drsResolverResponse
-  }
-
-  private[localizer] def toUriType(drsResponse: DrsResolverResponse) : Try[URIType] = {
-    (drsResponse.accessUrl, drsResponse.gsUri) match {
-      case (Some(accessUrl), _) =>
-        Try(URIType.HTTPS)
-      case (_, Some(gcsPath)) =>
-        Try(URIType.GCS)
-      case _ =>
-        Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
-    }
-  }
-
-  private[localizer] def bulkResolve(): Unit = {
-    // Load path resolver once from config
-    val resolver = getDrsPathResolver
-
-    // resolve all the URIs once.
-    val resolvedUris : Try[Map[DrsResolverResponse, String]] = for {
-      drsUriToDownloadLocationMap : Map[String, String] <- loadCSVAsMap("path/to/manifest")
-      resolvedUriToDownloadLocationMap : Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1), value._2))
-    } yield resolvedUriToDownloadLocationMap
-
-    // find the GCS ones and download them
-      //find the HTTPS ones and download them
-    val exitCode: IO[ExitCode] = for {
-      drsResponses <- resolvedUris
-      httpsUris = drsResponses.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
-      uriToDownloadLocation = httpsUris.map(drsResponseToDownloadLoc => (drsResponseToDownloadLoc._1.accessUrl.get.toString, drsResponseToDownloadLoc._2))
-      jsonString <- uriToDownloadLocation.toJson
-      pathOfJsonFile <- writeJsonManifest(generateJsonPathForManifest("path/to/manifest"), jsonString)
-      downloadSuccess <- downloadInParallel(pathOfJsonFile)
-    } yield downloadSuccess
-
-  }
-
 }
 
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
new file mode 100644
index 00000000000..11096d17bf5
--- /dev/null
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -0,0 +1,112 @@
+package drs.localizer.downloaders
+
+import cats.data.Validated.{Invalid, Valid}
+import cats.effect.{ExitCode, IO}
+import cloud.nio.impl.drs.AccessUrl
+import com.typesafe.scalalogging.StrictLogging
+import common.exception.AggregatedMessageException
+import common.util.StringUtil._
+import common.validation.ErrorOr.ErrorOr
+import drs.localizer.downloaders.AccessUrlDownloader._
+
+import scala.sys.process.{Process, ProcessLogger}
+import scala.util.Try
+import scala.util.matching.Regex
+
+case class GetmResult(returnCode: Int, stderr: String)
+
+case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String], hashes: Hashes) extends Downloader with StrictLogging {
+
+  def generateJsonManifest : Try[String] = {
+    //TODO: Make a JSON file with the structure:
+    //[
+    //  {
+    //      "url" : accessUrl,
+    //       "filepath" : containerPath,
+    //      "checksum" : "not sure what goes here",
+    //      "checksum-algorithm" : GetmChecksum(hashes, accessUrl)
+    //  },
+    //  {
+    //      "url" : accessUrl,
+    //      "filepath": containerPath,
+    //      "checksum" : "not sure what goes here",
+    //    "checksum-algorithm": GetmChecksum(hashes, accessUrl)
+    //  }
+    //]
+
+  }
+
+  def generateDownloadScript: ErrorOr[String] = {
+    val signedUrl = accessUrl.url
+    GetmChecksum(hashes, accessUrl).args map { checksumArgs =>
+      s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --filepath '$downloadLoc' '$signedUrl'"""
+    }
+  }
+
+  def runGetm: IO[GetmResult] = {
+    generateDownloadScript match {
+      case Invalid(errors) =>
+        IO.raiseError(AggregatedMessageException("Error generating access URL download script", errors.toList))
+      case Valid(script) => IO {
+        val copyCommand = Seq("bash", "-c", script)
+        val copyProcess = Process(copyCommand)
+
+        val stderr = new StringBuilder()
+        val errorCapture: String => Unit = { s => stderr.append(s); () }
+
+        // As of `getm` version 0.0.4 the contents of stdout do not appear to be interesting (only a progress bar
+        // with no option to suppress it), so ignore stdout for now. If stdout becomes interesting in future versions
+        // of `getm` it can be captured just like stderr is being captured here.
+        val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
+
+        GetmResult(returnCode, stderr.toString().trim())
+      }
+    }
+  }
+
+  override def download: IO[DownloadResult] = {
+    // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's
+    // workspace bucket, but that bucket may have visibility different than the data referenced by the signed URL.
+    val masked = accessUrl.url.maskSensitiveUri
+    logger.info(s"Attempting to download data to '$downloadLoc' from access URL '$masked'.")
+
+    runGetm map toDownloadResult
+  }
+
+  def toDownloadResult(getmResult: GetmResult): DownloadResult = {
+    getmResult match {
+      case GetmResult(0, stderr) if stderr.isEmpty =>
+        DownloadSuccess
+      case GetmResult(0, stderr) =>
+        stderr match {
+          case ChecksumFailureMessage() =>
+            ChecksumFailure
+          case _ =>
+            UnrecognizedRetryableDownloadFailure(ExitCode(0))
+        }
+      case GetmResult(rc, stderr) =>
+        stderr match {
+          case HttpStatusMessage(status) =>
+            Integer.parseInt(status) match {
+              case 408 | 429 =>
+                RecognizedRetryableDownloadFailure(ExitCode(rc))
+              case s if s / 100 == 4 =>
+                FatalDownloadFailure(ExitCode(rc))
+              case s if s / 100 == 5 =>
+                RecognizedRetryableDownloadFailure(ExitCode(rc))
+              case _ =>
+                UnrecognizedRetryableDownloadFailure(ExitCode(rc))
+            }
+          case _ =>
+            UnrecognizedRetryableDownloadFailure(ExitCode(rc))
+        }
+    }
+  }
+}
+
+object AccessUrlDownloader {
+  type Hashes = Option[Map[String, String]]
+
+  val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r
+  val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r
+}
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
index 8465ede0dd6..76fb1e426a9 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
@@ -7,6 +7,8 @@ import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 trait DownloaderFactory {
   def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader]
 
+  def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap : Map[AccessUrl, String], hashes: Hashes) : IO[Downloader]
+
   def buildGcsUriDownloader(gcsPath: String,
                             serviceAccountJsonOption: Option[String],
                             downloadLoc: String,

From 9e6b1f5aa70022c192ac7404a176201f02e4f1d3 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 8 Aug 2023 16:55:23 -0400
Subject: [PATCH 07/36] json

---
 .../downloaders/AccessUrlDownloader.scala     | 27 +++++++++++++++++++
 .../localizer/downloaders/GetmChecksum.scala  |  8 +++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
index ae6f2fa4f1e..8d360a91483 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
@@ -10,6 +10,7 @@ import common.validation.ErrorOr.ErrorOr
 import drs.localizer.downloaders.AccessUrlDownloader._
 
 import scala.sys.process.{Process, ProcessLogger}
+import scala.util.Try
 import scala.util.matching.Regex
 
 case class GetmResult(returnCode: Int, stderr: String)
@@ -22,6 +23,32 @@ case class AccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes
     }
   }
 
+  def toJsonString(accessUrl: String, filepath : String, checksum : ErrorOr[String], checksumAlgorithm : String): String = {
+    //todo: use flatmap
+    if (checksum.isValid) {
+      return s"""{
+                | "url" : "$accessUrl"
+                | "filepath" : "$filepath"
+                | "checksum" : "$checksum"
+                | "checksum-algorithm" : "$checksumAlgorithm"
+                | }
+                |""".stripMargin
+    }
+
+    s"""{
+         | "url" : "$accessUrl"
+         | "filepath" : "$filepath"
+         | }
+         |""".stripMargin
+  }
+  def generateJsonManifest(accessUrlToDownloadDest : Map[AccessUrl, String]) : Try[String] = {
+    var jsonString : String = "[\n"
+    for (pair <- accessUrlToDownloadDest)
+      jsonString ++ toJsonString(pair._1.url, pair._2, GetmChecksum(hashes, pair._1).escapedChecksum, GetmChecksum(hashes, pair._1).getmAlgorithm)
+
+    jsonString ++ "]"
+  }
+
   def runGetm: IO[GetmResult] = {
     generateDownloadScript match {
       case Invalid(errors) =>
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
index 2a39a6543a3..dfc84fc1949 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
@@ -13,13 +13,15 @@ sealed trait GetmChecksum {
   def getmAlgorithm: String
   def rawValue: String
   def value: ErrorOr[String] = rawValue.validNel
+  def escapedChecksum : ErrorOr[String] = {
+    value map { v => StringEscapeUtils.escapeXSI(v)}
+  }
   def args: ErrorOr[String] = {
     // The value for `--checksum-algorithm` is constrained by the algorithm names in the `sealed` hierarchy of
     // `GetmChecksum`, but the value for `--checksum` is largely a function of data returned by the DRS server.
     // Shell escape this to avoid injection.
-    value map { v =>
-      val escapedValue = StringEscapeUtils.escapeXSI(v)
-      s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedValue"
+    value map { _ =>
+      s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedChecksum"
     }
   }
 }

From 61f248eaf4cdcf1042fd9a9ef7d5d6937fbbad04 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Mon, 14 Aug 2023 10:14:08 -0400
Subject: [PATCH 08/36] stash

---
 .../drs/localizer/DrsLocalizerMain.scala      | 17 ++------
 .../downloaders/AccessUrlDownloader.scala     | 40 +++++++++++++++----
 2 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 9c7754b9944..8439c5c78f3 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -65,6 +65,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     IO.pure(ExitCode.Error)
   }
 
+  /*
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
     commandLineArguments.manifestPath match {
       case Some(manifestPath) =>
@@ -82,6 +83,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
         localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
     }
   }
+  */
 
   def getDrsPathResolver(drsCredentials: DrsCredentials): IO[DrsLocalizerDrsPathResolver] = {
     IO {
@@ -101,7 +103,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
         Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
     }
   }
-  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials, dummy : Boolean): IO[ExitCode] = {
+  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
     commandLineArguments.manifestPath match {
       case Some(manifestPath) => {
           val resolvedUris: Try[Map[DrsResolverResponse, String]] = for {
@@ -112,7 +114,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
 
           if(resolvedUris.isFailure) {
               return IO(ExitCode.Error) //Could not resolve DRS urls. Fail before attempting to download anything
-            }
+          }
 
           val accessUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
           val googleUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.GCS)
@@ -180,17 +182,6 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       if !map.isEmpty //An empty map should be considered an error - we're expecting at least one k/v pair in the manifest.
     } yield map
   }
-
-  //Write string to file. Returns the filepath that the string was written to.
-  private def writeJsonManifest(destinationPath : String, jsonData : JsValue): Try[String] = {
-    Try(Files.write(Paths.get(destinationPath), jsonData.toString().getBytes(StandardCharsets.UTF_8)).toString)
-  }
-  // We're going to write a JSON version of the CSV manifest we're provided.
-  // Save the new manifest to wherever the CSV is saved, but replace .csv with _manifest.json.
-  private def generateJsonPathForManifest(csvPath : String) : String = {
-    csvPath.substring(0, csvPath.lastIndexOf('.')) + "_manifest.json"
-  }
-
 }
 
 object URIType extends Enumeration {
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
index 8d360a91483..25cf88d53c9 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
@@ -3,6 +3,9 @@ package drs.localizer.downloaders
 import cats.data.Validated.{Invalid, Valid}
 import cats.effect.{ExitCode, IO}
 import cloud.nio.impl.drs.AccessUrl
+
+import java.nio.file.{Files, Path, Paths}
+import java.nio.charset.StandardCharsets
 import com.typesafe.scalalogging.StrictLogging
 import common.exception.AggregatedMessageException
 import common.util.StringUtil._
@@ -23,30 +26,51 @@ case class AccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes
     }
   }
 
+  def generateBulkDownloadScript: ErrorOr[String] = {
+    val manifestPath = generateJsonManifest()
+  }
+
   def toJsonString(accessUrl: String, filepath : String, checksum : ErrorOr[String], checksumAlgorithm : String): String = {
-    //todo: use flatmap
+    //NB: trailing comma is being removed in generateJsonManifest
     if (checksum.isValid) {
       return s"""{
                 | "url" : "$accessUrl"
                 | "filepath" : "$filepath"
                 | "checksum" : "$checksum"
                 | "checksum-algorithm" : "$checksumAlgorithm"
-                | }
+                | },
                 |""".stripMargin
     }
 
     s"""{
          | "url" : "$accessUrl"
          | "filepath" : "$filepath"
-         | }
+         | },
          |""".stripMargin
   }
-  def generateJsonManifest(accessUrlToDownloadDest : Map[AccessUrl, String]) : Try[String] = {
+  def generateJsonManifest(accessUrlToDownloadDest : Map[AccessUrl, String]) : Try[Path] = {
+    //write a json file that looks like:
+    // [
+    //  {
+    //     "url" : "www.123.com"
+    //     "filepath" : "path/to/where/123/should/be/downloaded"
+    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf"
+    //     "checksum-algorithm" : "md5"
+    //  },
+    //  {
+    //     "url" : "www.567.com"
+    //     "filepath" : "path/to/where/567/should/be/downloaded"
+    //     "checksum" : "asdasdasfsdfsdfasdsdfasd"
+    //     "checksum-algorithm" : "md5"
+    //  }
+    // ]
     var jsonString : String = "[\n"
-    for (pair <- accessUrlToDownloadDest)
-      jsonString ++ toJsonString(pair._1.url, pair._2, GetmChecksum(hashes, pair._1).escapedChecksum, GetmChecksum(hashes, pair._1).getmAlgorithm)
-
-    jsonString ++ "]"
+    for ( (accessUrl -> downloadDestination) <- accessUrlToDownloadDest) {
+      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).escapedChecksum, GetmChecksum(hashes, accessUrl).getmAlgorithm)
+    }
+    jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
+    jsonString += "]"
+    Try(Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8)))
   }
 
   def runGetm: IO[GetmResult] = {

From 125202ce575b19332ef4b865c65801825f6f08a0 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Mon, 14 Aug 2023 10:15:08 -0400
Subject: [PATCH 09/36] expired tokens

---
 src/ci/resources/tes_application.conf | 178 ++++++++++++++++++++++----
 1 file changed, 151 insertions(+), 27 deletions(-)

diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf
index 5eb812a52d4..91c740ce52d 100644
--- a/src/ci/resources/tes_application.conf
+++ b/src/ci/resources/tes_application.conf
@@ -2,26 +2,43 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
 filesystems {
-  blob {
-    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-    global {
-      # One BFSM is shared across all BlobPathBuilders
-      class = "cromwell.filesystems.blob.BlobFileSystemManager"
-      config {
-        container: "cromwell"
-        endpoint: "https://<storage-account>.blob.core.windows.net"
-        subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
-        # WSM config is needed for accessing WSM-managed blob containers
-        # created in Terra workspaces.
-        workspace-manager {
-          url: "https://workspace.dsde-dev.broadinstitute.org"
-          workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          b2cToken: "Zardoz"
+
+    blob {
+        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+        global {
+          class = "cromwell.filesystems.blob.BlobFileSystemManager"
+          config {
+            workspace-manager: {
+              url: "https://workspace.dsde-dev.broadinstitute.org"
+              workspace-id: "d4564046-bbba-495c-afec-14f7d3a8283a"
+              container-resource-id: "9e7bdba5-047e-430c-810d-c7cb6645c155"
+              b2cToken: "eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlEMU5hdF9YWldYMDg0Z1lxU1BHWWpjOWVTMlNEcFJhRVJNOFprVjJWc25aUU9QMndrWEJycmVQQWlwWGMzWXJwYUhQanhoVmEwRWk1NTlGTmpkZExwVHBXY0Y4SFNlTHJiTnZqaWNCYUo3cVZMWUR3aThvdXVrMzNLVjJ6V3RrcWY3YWhueVRtZVVxaEo4aG1FUTlwOHk0c3VaOTAyMkNHQWFDZ1lLQVIwU0FSQVNGUUhzdllsc3otXzctSlNrbGx6Zld3V3Y1OWtOdVEwMTc0IiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTE3NjE1NDUsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MTc2NTE0NSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkxNzYxNTQ1fQ.k8XqUUWWfsJqVru8AdD8PqMw5js0Dlp7vtKWG2HxUDCNuQoz9ERs-QbvWU2XHmQZ2kS4zVtTzDoTc4PbNJ_Dy81I9nNDczdMxsyhKZ1GhJHiuS_h6nFaweLQTKCfrzye10VJdZlyR0cfBllbu7ct6-S91AvpTbwxKmU2p33eBw4ToEwdYNrDkbrIxmUHwrVN0nmoxQcIcouD0pOMcpR_Ad6hYWn1s9ug3BguM6jAhhjXWxJj4DIdcJ4mnDDhKkWbcV-NdphNVSl1COAY7hstEuHMK_3-Of6l1SMB9I-WA-M1esDmsXAcL1y8g0DTIdzDkEqrrxOb4k-NbQCgMiMElw"
+              }
+            container: "sc-d4564046-bbba-495c-afec-14f7d3a8283a"
+            endpoint: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net"
+          }
         }
       }
-    }
-  }
+#   blob {
+#     class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+#     global {
+#       # One BFSM is shared across all BlobPathBuilders
+#       class = "cromwell.filesystems.blob.BlobFileSystemManager"
+#       config {
+#         container: "cromwell"
+#         endpoint: "https://coaexternalstorage.blob.core.windows.net"
+#         subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9"
+#         # WSM config is needed for accessing WSM-managed blob containers
+#         # created in Terra workspaces.
+#         workspace-manager {
+#           url: "https://workspace.dsde-dev.broadinstitute.org"
+#           workspace-id: "ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c"
+#           container-resource-id: "983bcc2b-3134-48d0-b52b-5df1fe09d161"
+#           b2cToken: "Bearer eyJhbGciOiJSUzI1NiIsImt...Jw49ZA"
+#         }
+#       }
+#     }
+#   }
 }
 
 engine {
@@ -33,7 +50,7 @@ engine {
       enabled: true
     }
     blob {
-      enabled: false
+      enabled: true
     }
   }
 }
@@ -46,26 +63,30 @@ backend {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
         # Use for running on blob storage
-        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
-        root = "cromwell-executions"
+        #root = "https://coaexternalstorage.blob.core.windows.net/cromwell/janet-20230505"
+        root = "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/"
+        #root = "cromwell-executions"
         dockerRoot = "/cromwell-executions"
-        endpoint = "http://127.0.0.1:9000/v1/tasks"
+        endpoint = "https://lzd98e904c9c1cee247af683bc764eb1b22f3912f8e7839afe.servicebus.windows.net/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/tes/v1/tasks"
         concurrent-job-limit = 1000
-
+        workflow-execution-identity = "pet-2676571657071603a9eab"
+        bearer-token= "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlEMU5hdF9YWldYMDg0Z1lxU1BHWWpjOWVTMlNEcFJhRVJNOFprVjJWc25aUU9QMndrWEJycmVQQWlwWGMzWXJwYUhQanhoVmEwRWk1NTlGTmpkZExwVHBXY0Y4SFNlTHJiTnZqaWNCYUo3cVZMWUR3aThvdXVrMzNLVjJ6V3RrcWY3YWhueVRtZVVxaEo4aG1FUTlwOHk0c3VaOTAyMkNHQWFDZ1lLQVIwU0FSQVNGUUhzdllsc3otXzctSlNrbGx6Zld3V3Y1OWtOdVEwMTc0IiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTE3NjE1NDUsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MTc2NTE0NSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkxNzYxNTQ1fQ.k8XqUUWWfsJqVru8AdD8PqMw5js0Dlp7vtKWG2HxUDCNuQoz9ERs-QbvWU2XHmQZ2kS4zVtTzDoTc4PbNJ_Dy81I9nNDczdMxsyhKZ1GhJHiuS_h6nFaweLQTKCfrzye10VJdZlyR0cfBllbu7ct6-S91AvpTbwxKmU2p33eBw4ToEwdYNrDkbrIxmUHwrVN0nmoxQcIcouD0pOMcpR_Ad6hYWn1s9ug3BguM6jAhhjXWxJj4DIdcJ4mnDDhKkWbcV-NdphNVSl1COAY7hstEuHMK_3-Of6l1SMB9I-WA-M1esDmsXAcL1y8g0DTIdzDkEqrrxOb4k-NbQCgMiMElw"
         # Override default polling to make it faster for speedy tests
         poll-backoff {
-          min: "1 seconds"
+          min: "10 seconds"
           max: "5 minutes"
           multiplier: 1.1
           randomization-factor: 0.5
         }
-        
+
+        use_tes_11_preview_backend_parameters = true
+
         filesystems {
           blob {
-            enabled: false
+            enabled: true
           }
           local {
-            enabled: true
+            enabled: false
           }
           http {
             enabled: true
@@ -78,3 +99,106 @@ backend {
     }
   }
 }
+
+workflow-options {
+  # Directory where to write per workflow logs
+  workflow-log-dir: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs"
+}
+
+# call-caching {
+#   # Allows re-use of existing results for jobs you've already run
+#   # (default: false)
+#   enabled = true
+#
+#   # Whether to invalidate a cache result forever if we cannot reuse them. Disable this if you expect some cache copies
+#   # to fail for external reasons which should not invalidate the cache (e.g. auth differences between users):
+#   # (default: true)
+#   invalidate-bad-cache-results = false
+#
+#   # The maximum number of times Cromwell will attempt to copy cache hits before giving up and running the job.
+#   max-failed-copy-attempts = 10
+# }
+
+
+
+# ////////////
+# include required(classpath("application.conf"))
+# include "build_application.inc.conf"
+#
+# filesystems {
+#   blob {
+#     class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+#     global {
+#       # One BFSM is shared across all BlobPathBuilders
+#       class = "cromwell.filesystems.blob.BlobFileSystemManager"
+#       config {
+#         container: "cromwell"
+#         endpoint: "https://<storage-account>.blob.core.windows.net"
+#         subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
+#         # WSM config is needed for accessing WSM-managed blob containers
+#         # created in Terra workspaces.
+#         workspace-manager {
+#           url: "https://workspace.dsde-dev.broadinstitute.org"
+#           workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+#           container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+#           b2cToken: "Zardoz"
+#         }
+#       }
+#     }
+#   }
+# }
+#
+# engine {
+#   filesystems {
+#     local {
+#       enabled: true
+#     }
+#     http {
+#       enabled: true
+#     }
+#     blob {
+#       enabled: false
+#     }
+#   }
+# }
+#
+#
+# backend {
+#   default = "TES"
+#   providers {
+#     TES {
+#       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+#       config {
+#         # Use for running on blob storage
+#         #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
+#         root = "cromwell-executions"
+#         dockerRoot = "/cromwell-executions"
+#         endpoint = "http://127.0.0.1:9000/v1/tasks"
+#         concurrent-job-limit = 1000
+#
+#         # Override default polling to make it faster for speedy tests
+#         poll-backoff {
+#           min: "1 seconds"
+#           max: "5 minutes"
+#           multiplier: 1.1
+#           randomization-factor: 0.5
+#         }
+#
+#         filesystems {
+#           blob {
+#             enabled: false
+#           }
+#           local {
+#             enabled: true
+#           }
+#           http {
+#             enabled: true
+#           }
+#         }
+#       }
+#
+#       # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
+#       include "dockerhub_provider_config_v1.inc.conf"
+#     }
+#   }
+# }

From 28e1b63dabe28fca9dd3d91a43b3d5095d7f5998 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 15 Aug 2023 16:21:21 -0400
Subject: [PATCH 10/36] think about stuff

---
 .../src/main/scala/drs/localizer/DrsLocalizerMain.scala   | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 8439c5c78f3..cbc78f3bb3d 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -104,11 +104,12 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
+    //todo: just fail here if this can't be gotten
+    val resolver = getDrsPathResolver(drsCredentials).unsafeRunSync()
     commandLineArguments.manifestPath match {
       case Some(manifestPath) => {
           val resolvedUris: Try[Map[DrsResolverResponse, String]] = for {
-            resolver: IO[DrsLocalizerDrsPathResolver] <- getDrsPathResolver(drsCredentials)
-            resolvedDrsUris: Try[Map[DrsResolverResponse, String]] <- resolveCSVManifest(manifestPath, resolver)
+            resolvedDrsUris: Map[DrsResolverResponse, String] <- resolveCSVManifest(manifestPath, resolver)
             if !resolvedDrsUris.isEmpty
           } yield resolvedDrsUris
 
@@ -122,7 +123,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
           val bulkAccessResult : IO[ExitCode] = bulkDownloadResolvedAccessUrls(accessUrls, drsCredentials)
           val googleResult : List[IO[DownloadResult]] = bulkDownloadResolvedGoogleUrls(googleUrls, commandLineArguments.googleRequesterPaysProject)
 
-          //TODO: return an appropriate exit code
+        IO(ExitCode.Success)
         }
       case None =>
         val drsObject = commandLineArguments.drsObject.get
@@ -133,6 +134,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
 
   def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
     //TODO: invoke new bulk getm downloader
+    return IO{ExitCode(0)}
   }
 
   def bulkDownloadResolvedGoogleUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], googleRequesterPaysProject : Option[String]) : List[IO[DownloadResult]] = {

From 6e9b698a053d5f9924a981042e305abc4b2b66a2 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 15 Aug 2023 16:25:39 -0400
Subject: [PATCH 11/36] undo sins

---
 .../downloaders/AccessUrlDownloader.scala     |  47 -----
 .../downloaders/BulkAccessUrlDownloader.scala |  49 +++++
 .../localizer/downloaders/GetmChecksum.scala  |   8 +-
 project/Dependencies.scala                    |   1 -
 src/ci/resources/tes_application.conf         | 178 +++---------------
 .../TesAsyncBackendJobExecutionActor.scala    |   5 +-
 6 files changed, 81 insertions(+), 207 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
index 25cf88d53c9..5d1099e31bc 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
@@ -26,53 +26,6 @@ case class AccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes
     }
   }
 
-  def generateBulkDownloadScript: ErrorOr[String] = {
-    val manifestPath = generateJsonManifest()
-  }
-
-  def toJsonString(accessUrl: String, filepath : String, checksum : ErrorOr[String], checksumAlgorithm : String): String = {
-    //NB: trailing comma is being removed in generateJsonManifest
-    if (checksum.isValid) {
-      return s"""{
-                | "url" : "$accessUrl"
-                | "filepath" : "$filepath"
-                | "checksum" : "$checksum"
-                | "checksum-algorithm" : "$checksumAlgorithm"
-                | },
-                |""".stripMargin
-    }
-
-    s"""{
-         | "url" : "$accessUrl"
-         | "filepath" : "$filepath"
-         | },
-         |""".stripMargin
-  }
-  def generateJsonManifest(accessUrlToDownloadDest : Map[AccessUrl, String]) : Try[Path] = {
-    //write a json file that looks like:
-    // [
-    //  {
-    //     "url" : "www.123.com"
-    //     "filepath" : "path/to/where/123/should/be/downloaded"
-    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf"
-    //     "checksum-algorithm" : "md5"
-    //  },
-    //  {
-    //     "url" : "www.567.com"
-    //     "filepath" : "path/to/where/567/should/be/downloaded"
-    //     "checksum" : "asdasdasfsdfsdfasdsdfasd"
-    //     "checksum-algorithm" : "md5"
-    //  }
-    // ]
-    var jsonString : String = "[\n"
-    for ( (accessUrl -> downloadDestination) <- accessUrlToDownloadDest) {
-      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).escapedChecksum, GetmChecksum(hashes, accessUrl).getmAlgorithm)
-    }
-    jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
-    jsonString += "]"
-    Try(Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8)))
-  }
-
   def runGetm: IO[GetmResult] = {
     generateDownloadScript match {
       case Invalid(errors) =>
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index 11096d17bf5..fc418034aa6 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -43,6 +43,55 @@ case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String
     }
   }
 
+  def generateBulkDownloadScript: ErrorOr[String] = {
+    val manifestPath = generateJsonManifest()
+  }
+
+  def toJsonString(accessUrl: String, filepath: String, checksum: ErrorOr[String], checksumAlgorithm: String): String = {
+    //NB: trailing comma is being removed in generateJsonManifest
+    if (checksum.isValid) {
+      return
+        s"""{
+           | "url" : "$accessUrl"
+           | "filepath" : "$filepath"
+           | "checksum" : "$checksum"
+           | "checksum-algorithm" : "$checksumAlgorithm"
+           | },
+           |""".stripMargin
+    }
+
+    s"""{
+       | "url" : "$accessUrl"
+       | "filepath" : "$filepath"
+       | },
+       |""".stripMargin
+  }
+
+  def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
+    //write a json file that looks like:
+    // [
+    //  {
+    //     "url" : "www.123.com"
+    //     "filepath" : "path/to/where/123/should/be/downloaded"
+    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf"
+    //     "checksum-algorithm" : "md5"
+    //  },
+    //  {
+    //     "url" : "www.567.com"
+    //     "filepath" : "path/to/where/567/should/be/downloaded"
+    //     "checksum" : "asdasdasfsdfsdfasdsdfasd"
+    //     "checksum-algorithm" : "md5"
+    //  }
+    // ]
+    var jsonString: String = "[\n"
+    for ((accessUrl -> downloadDestination) <- accessUrlToDownloadDest) {
+      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).escapedChecksum, GetmChecksum(hashes, accessUrl).getmAlgorithm)
+    }
+    jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
+    jsonString += "]"
+    Try(Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8)))
+  }
+
   def runGetm: IO[GetmResult] = {
     generateDownloadScript match {
       case Invalid(errors) =>
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
index dfc84fc1949..2a39a6543a3 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
@@ -13,15 +13,13 @@ sealed trait GetmChecksum {
   def getmAlgorithm: String
   def rawValue: String
   def value: ErrorOr[String] = rawValue.validNel
-  def escapedChecksum : ErrorOr[String] = {
-    value map { v => StringEscapeUtils.escapeXSI(v)}
-  }
   def args: ErrorOr[String] = {
     // The value for `--checksum-algorithm` is constrained by the algorithm names in the `sealed` hierarchy of
     // `GetmChecksum`, but the value for `--checksum` is largely a function of data returned by the DRS server.
     // Shell escape this to avoid injection.
-    value map { _ =>
-      s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedChecksum"
+    value map { v =>
+      val escapedValue = StringEscapeUtils.escapeXSI(v)
+      s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedValue"
     }
   }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index d48e24a0a53..b2e94cc7470 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -652,7 +652,6 @@ object Dependencies {
     "com.softwaremill.sttp" %% "circe" % sttpV,
     "com.github.scopt" %% "scopt" % scoptV,
     "org.apache.commons" % "commons-csv" % commonsCsvV,
-    "org.gnieh" %% "diffson-spray-json" % diffsonSprayJsonV
   ) ++ circeDependencies ++ catsDependencies ++ slf4jBindingDependencies ++ languageFactoryDependencies ++ azureDependencies
 
   val allProjectDependencies: List[ModuleID] =
diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf
index 91c740ce52d..5eb812a52d4 100644
--- a/src/ci/resources/tes_application.conf
+++ b/src/ci/resources/tes_application.conf
@@ -2,43 +2,26 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
 filesystems {
-
-    blob {
-        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-        global {
-          class = "cromwell.filesystems.blob.BlobFileSystemManager"
-          config {
-            workspace-manager: {
-              url: "https://workspace.dsde-dev.broadinstitute.org"
-              workspace-id: "d4564046-bbba-495c-afec-14f7d3a8283a"
-              container-resource-id: "9e7bdba5-047e-430c-810d-c7cb6645c155"
-              b2cToken: "eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlEMU5hdF9YWldYMDg0Z1lxU1BHWWpjOWVTMlNEcFJhRVJNOFprVjJWc25aUU9QMndrWEJycmVQQWlwWGMzWXJwYUhQanhoVmEwRWk1NTlGTmpkZExwVHBXY0Y4SFNlTHJiTnZqaWNCYUo3cVZMWUR3aThvdXVrMzNLVjJ6V3RrcWY3YWhueVRtZVVxaEo4aG1FUTlwOHk0c3VaOTAyMkNHQWFDZ1lLQVIwU0FSQVNGUUhzdllsc3otXzctSlNrbGx6Zld3V3Y1OWtOdVEwMTc0IiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTE3NjE1NDUsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MTc2NTE0NSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkxNzYxNTQ1fQ.k8XqUUWWfsJqVru8AdD8PqMw5js0Dlp7vtKWG2HxUDCNuQoz9ERs-QbvWU2XHmQZ2kS4zVtTzDoTc4PbNJ_Dy81I9nNDczdMxsyhKZ1GhJHiuS_h6nFaweLQTKCfrzye10VJdZlyR0cfBllbu7ct6-S91AvpTbwxKmU2p33eBw4ToEwdYNrDkbrIxmUHwrVN0nmoxQcIcouD0pOMcpR_Ad6hYWn1s9ug3BguM6jAhhjXWxJj4DIdcJ4mnDDhKkWbcV-NdphNVSl1COAY7hstEuHMK_3-Of6l1SMB9I-WA-M1esDmsXAcL1y8g0DTIdzDkEqrrxOb4k-NbQCgMiMElw"
-              }
-            container: "sc-d4564046-bbba-495c-afec-14f7d3a8283a"
-            endpoint: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net"
-          }
+  blob {
+    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+    global {
+      # One BFSM is shared across all BlobPathBuilders
+      class = "cromwell.filesystems.blob.BlobFileSystemManager"
+      config {
+        container: "cromwell"
+        endpoint: "https://<storage-account>.blob.core.windows.net"
+        subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
+        # WSM config is needed for accessing WSM-managed blob containers
+        # created in Terra workspaces.
+        workspace-manager {
+          url: "https://workspace.dsde-dev.broadinstitute.org"
+          workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+          container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+          b2cToken: "Zardoz"
         }
       }
-#   blob {
-#     class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-#     global {
-#       # One BFSM is shared across all BlobPathBuilders
-#       class = "cromwell.filesystems.blob.BlobFileSystemManager"
-#       config {
-#         container: "cromwell"
-#         endpoint: "https://coaexternalstorage.blob.core.windows.net"
-#         subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9"
-#         # WSM config is needed for accessing WSM-managed blob containers
-#         # created in Terra workspaces.
-#         workspace-manager {
-#           url: "https://workspace.dsde-dev.broadinstitute.org"
-#           workspace-id: "ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c"
-#           container-resource-id: "983bcc2b-3134-48d0-b52b-5df1fe09d161"
-#           b2cToken: "Bearer eyJhbGciOiJSUzI1NiIsImt...Jw49ZA"
-#         }
-#       }
-#     }
-#   }
+    }
+  }
 }
 
 engine {
@@ -50,7 +33,7 @@ engine {
       enabled: true
     }
     blob {
-      enabled: true
+      enabled: false
     }
   }
 }
@@ -63,30 +46,26 @@ backend {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
         # Use for running on blob storage
-        #root = "https://coaexternalstorage.blob.core.windows.net/cromwell/janet-20230505"
-        root = "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/"
-        #root = "cromwell-executions"
+        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
+        root = "cromwell-executions"
         dockerRoot = "/cromwell-executions"
-        endpoint = "https://lzd98e904c9c1cee247af683bc764eb1b22f3912f8e7839afe.servicebus.windows.net/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/tes/v1/tasks"
+        endpoint = "http://127.0.0.1:9000/v1/tasks"
         concurrent-job-limit = 1000
-        workflow-execution-identity = "pet-2676571657071603a9eab"
-        bearer-token= "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlEMU5hdF9YWldYMDg0Z1lxU1BHWWpjOWVTMlNEcFJhRVJNOFprVjJWc25aUU9QMndrWEJycmVQQWlwWGMzWXJwYUhQanhoVmEwRWk1NTlGTmpkZExwVHBXY0Y4SFNlTHJiTnZqaWNCYUo3cVZMWUR3aThvdXVrMzNLVjJ6V3RrcWY3YWhueVRtZVVxaEo4aG1FUTlwOHk0c3VaOTAyMkNHQWFDZ1lLQVIwU0FSQVNGUUhzdllsc3otXzctSlNrbGx6Zld3V3Y1OWtOdVEwMTc0IiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTE3NjE1NDUsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MTc2NTE0NSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkxNzYxNTQ1fQ.k8XqUUWWfsJqVru8AdD8PqMw5js0Dlp7vtKWG2HxUDCNuQoz9ERs-QbvWU2XHmQZ2kS4zVtTzDoTc4PbNJ_Dy81I9nNDczdMxsyhKZ1GhJHiuS_h6nFaweLQTKCfrzye10VJdZlyR0cfBllbu7ct6-S91AvpTbwxKmU2p33eBw4ToEwdYNrDkbrIxmUHwrVN0nmoxQcIcouD0pOMcpR_Ad6hYWn1s9ug3BguM6jAhhjXWxJj4DIdcJ4mnDDhKkWbcV-NdphNVSl1COAY7hstEuHMK_3-Of6l1SMB9I-WA-M1esDmsXAcL1y8g0DTIdzDkEqrrxOb4k-NbQCgMiMElw"
+
         # Override default polling to make it faster for speedy tests
         poll-backoff {
-          min: "10 seconds"
+          min: "1 seconds"
           max: "5 minutes"
           multiplier: 1.1
           randomization-factor: 0.5
         }
-
-        use_tes_11_preview_backend_parameters = true
-
+        
         filesystems {
           blob {
-            enabled: true
+            enabled: false
           }
           local {
-            enabled: false
+            enabled: true
           }
           http {
             enabled: true
@@ -99,106 +78,3 @@ backend {
     }
   }
 }
-
-workflow-options {
-  # Directory where to write per workflow logs
-  workflow-log-dir: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs"
-}
-
-# call-caching {
-#   # Allows re-use of existing results for jobs you've already run
-#   # (default: false)
-#   enabled = true
-#
-#   # Whether to invalidate a cache result forever if we cannot reuse them. Disable this if you expect some cache copies
-#   # to fail for external reasons which should not invalidate the cache (e.g. auth differences between users):
-#   # (default: true)
-#   invalidate-bad-cache-results = false
-#
-#   # The maximum number of times Cromwell will attempt to copy cache hits before giving up and running the job.
-#   max-failed-copy-attempts = 10
-# }
-
-
-
-# ////////////
-# include required(classpath("application.conf"))
-# include "build_application.inc.conf"
-#
-# filesystems {
-#   blob {
-#     class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-#     global {
-#       # One BFSM is shared across all BlobPathBuilders
-#       class = "cromwell.filesystems.blob.BlobFileSystemManager"
-#       config {
-#         container: "cromwell"
-#         endpoint: "https://<storage-account>.blob.core.windows.net"
-#         subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
-#         # WSM config is needed for accessing WSM-managed blob containers
-#         # created in Terra workspaces.
-#         workspace-manager {
-#           url: "https://workspace.dsde-dev.broadinstitute.org"
-#           workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-#           container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-#           b2cToken: "Zardoz"
-#         }
-#       }
-#     }
-#   }
-# }
-#
-# engine {
-#   filesystems {
-#     local {
-#       enabled: true
-#     }
-#     http {
-#       enabled: true
-#     }
-#     blob {
-#       enabled: false
-#     }
-#   }
-# }
-#
-#
-# backend {
-#   default = "TES"
-#   providers {
-#     TES {
-#       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
-#       config {
-#         # Use for running on blob storage
-#         #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
-#         root = "cromwell-executions"
-#         dockerRoot = "/cromwell-executions"
-#         endpoint = "http://127.0.0.1:9000/v1/tasks"
-#         concurrent-job-limit = 1000
-#
-#         # Override default polling to make it faster for speedy tests
-#         poll-backoff {
-#           min: "1 seconds"
-#           max: "5 minutes"
-#           multiplier: 1.1
-#           randomization-factor: 0.5
-#         }
-#
-#         filesystems {
-#           blob {
-#             enabled: false
-#           }
-#           local {
-#             enabled: true
-#           }
-#           http {
-#             enabled: true
-#           }
-#         }
-#       }
-#
-#       # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
-#       include "dockerhub_provider_config_v1.inc.conf"
-#     }
-#   }
-# }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
index e46212817c4..7ff507e63f2 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -161,9 +161,8 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
         jobShell,
         mode)
     })
-    val testask = tesTask.map(TesTask.makeTask)
-    jobLogger.info(testask.toString)
-    testask
+
+    tesTask.map(TesTask.makeTask)
   }
 
   def writeScriptFile(): Future[Unit] = {

From a9a9c38671c518c10eecbe01b31e15e633640a6e Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 15 Aug 2023 16:27:00 -0400
Subject: [PATCH 12/36] undo one more sin

---
 .../scala/drs/localizer/downloaders/AccessUrlDownloader.scala | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
index 5d1099e31bc..ae6f2fa4f1e 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
@@ -3,9 +3,6 @@ package drs.localizer.downloaders
 import cats.data.Validated.{Invalid, Valid}
 import cats.effect.{ExitCode, IO}
 import cloud.nio.impl.drs.AccessUrl
-
-import java.nio.file.{Files, Path, Paths}
-import java.nio.charset.StandardCharsets
 import com.typesafe.scalalogging.StrictLogging
 import common.exception.AggregatedMessageException
 import common.util.StringUtil._
@@ -13,7 +10,6 @@ import common.validation.ErrorOr.ErrorOr
 import drs.localizer.downloaders.AccessUrlDownloader._
 
 import scala.sys.process.{Process, ProcessLogger}
-import scala.util.Try
 import scala.util.matching.Regex
 
 case class GetmResult(returnCode: Int, stderr: String)

From e935cfc6ac5447bf7ee27eb5d58adf48ac4b14fe Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 31 Aug 2023 14:24:05 -0400
Subject: [PATCH 13/36] deck chairs

---
 .../drs/localizer/DrsLocalizerMain.scala      |  46 +-
 .../downloaders/BulkAccessUrlDownloader.scala |  88 +--
 .../downloaders/DownloaderFactory.scala       |   2 +-
 ...w.194f43e1-319b-402d-9c1d-ff704a1db2a2.log | 585 ---------------
 ...w.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log | 670 ------------------
 5 files changed, 51 insertions(+), 1340 deletions(-)
 delete mode 100644 https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
 delete mode 100644 https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index cbc78f3bb3d..37e99ed4d32 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -2,7 +2,6 @@ package drs.localizer
 
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO, IOApp}
-import cats.implicits._
 import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition}
 import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
@@ -12,18 +11,14 @@ import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
 
-import java.io.{File, IOException}
+import java.io.{File}
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
-import spray.json._
-import DefaultJsonProtocol._
-import drs.localizer.URIType.{GCS, URIType}
+import drs.localizer.URIType.{URIType}
 
-import java.nio.file.{Files, Paths}
-import java.nio.charset.StandardCharsets
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Try}
 
 object DrsLocalizerMain extends IOApp with StrictLogging {
 
@@ -49,14 +44,15 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
 
   val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory {
-    override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] =
-      IO.pure(AccessUrlDownloader(accessUrl, downloadLoc, hashes))
-
+    override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
+      val downloader = AccessUrlDownloader(accessUrl, downloadLoc, hashes)
+      IO.pure(downloader)
+    }
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
       IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
 
-    override def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap: Map[AccessUrl, String], hashes: Hashes): IO[Downloader] = {
-      IO.pure(BulkAccessUrlDownloader(urlToDownloadDestinationMap, hashes))
+    override def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap: Map[AccessUrl, String]): IO[Downloader] = {
+      IO.pure(BulkAccessUrlDownloader(urlToDownloadDestinationMap))
     }
   }
 
@@ -95,9 +91,9 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
 
   private[localizer] def toUriType(drsResponse: DrsResolverResponse): Try[URIType] = {
     (drsResponse.accessUrl, drsResponse.gsUri) match {
-      case (Some(accessUrl), _) =>
+      case (Some(_), _) =>
         Try(URIType.HTTPS)
-      case (_, Some(gcsPath)) =>
+      case (_, Some(_)) =>
         Try(URIType.GCS)
       case _ =>
         Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
@@ -123,7 +119,12 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
           val bulkAccessResult : IO[ExitCode] = bulkDownloadResolvedAccessUrls(accessUrls, drsCredentials)
           val googleResult : List[IO[DownloadResult]] = bulkDownloadResolvedGoogleUrls(googleUrls, commandLineArguments.googleRequesterPaysProject)
 
-        IO(ExitCode.Success)
+        if(bulkAccessResult.equals(true) && googleResult.equals(true)) {
+          IO(ExitCode.Success)
+        } else {
+          IO(ExitCode.Error)
+        }
+
         }
       case None =>
         val drsObject = commandLineArguments.drsObject.get
@@ -133,8 +134,10 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
   }
 
   def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
-    //TODO: invoke new bulk getm downloader
-    return IO{ExitCode(0)}
+    val accessUrlMap = resolvedUrlToOutputDestination.map( pair => pair._1.accessUrl.get -> pair._2)
+    val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(accessUrlMap)
+    downloader.flatMap(_.download)
+    IO(ExitCode.Success) //tood
   }
 
   def bulkDownloadResolvedGoogleUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], googleRequesterPaysProject : Option[String]) : List[IO[DownloadResult]] = {
@@ -157,10 +160,9 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
   }
   // resolve all the URIs once.
   def resolveCSVManifest(csvManifestPath: String, resolver: DrsLocalizerDrsPathResolver) : Try[Map[DrsResolverResponse, String]] = {
-      for {
-        drsUriToDownloadLocationMap: Map[String, String] <- loadCSVAsMap(csvManifestPath)
-        resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1), value._2))
-      } yield resolvedUriToDownloadLocationMap
+    val drsUriToDownloadLocationMap: Map[String, String] = loadCSVAsMap(csvManifestPath).get
+    val resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1).unsafeRunSync(), value._2))
+    Try(resolvedUriToDownloadLocationMap)
     }
 
   private def localizeFile(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials, drsObject: String, containerPath: String) = {
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index fc418034aa6..fe6687d4b8c 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -1,56 +1,31 @@
 package drs.localizer.downloaders
 
-import cats.data.Validated.{Invalid, Valid}
 import cats.effect.{ExitCode, IO}
 import cloud.nio.impl.drs.AccessUrl
 import com.typesafe.scalalogging.StrictLogging
-import common.exception.AggregatedMessageException
-import common.util.StringUtil._
 import common.validation.ErrorOr.ErrorOr
 import drs.localizer.downloaders.AccessUrlDownloader._
 
+import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Path, Paths}
 import scala.sys.process.{Process, ProcessLogger}
 import scala.util.Try
 import scala.util.matching.Regex
 
 case class GetmResult(returnCode: Int, stderr: String)
 
-case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String], hashes: Hashes) extends Downloader with StrictLogging {
+case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String]) extends Downloader with StrictLogging {
 
-  def generateJsonManifest : Try[String] = {
-    //TODO: Make a JSON file with the structure:
-    //[
-    //  {
-    //      "url" : accessUrl,
-    //       "filepath" : containerPath,
-    //      "checksum" : "not sure what goes here",
-    //      "checksum-algorithm" : GetmChecksum(hashes, accessUrl)
-    //  },
-    //  {
-    //      "url" : accessUrl,
-    //      "filepath": containerPath,
-    //      "checksum" : "not sure what goes here",
-    //    "checksum-algorithm": GetmChecksum(hashes, accessUrl)
-    //  }
-    //]
-
-  }
-
-  def generateDownloadScript: ErrorOr[String] = {
-    val signedUrl = accessUrl.url
-    GetmChecksum(hashes, accessUrl).args map { checksumArgs =>
-      s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --filepath '$downloadLoc' '$signedUrl'"""
-    }
+  private def generateBulkDownloadScript: Try[String] = {
+    val manifestPath = generateJsonManifest(urlToDownloadLocation)
+    val downloadLoc = "getmDL"
+    val checksumArgs = "pleaseCompile"
+    generateJsonManifest(urlToDownloadLocation).map(mainfestPath =>  s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --manifest $manifestPath'""")
   }
 
-  def generateBulkDownloadScript: ErrorOr[String] = {
-    val manifestPath = generateJsonManifest()
-  }
-
-  def toJsonString(accessUrl: String, filepath: String, checksum: ErrorOr[String], checksumAlgorithm: String): String = {
+  private def toJsonString(accessUrl: String, filepath: String, checksum: ErrorOr[String], checksumAlgorithm: String): String = {
     //NB: trailing comma is being removed in generateJsonManifest
     if (checksum.isValid) {
-      return
         s"""{
            | "url" : "$accessUrl"
            | "filepath" : "$filepath"
@@ -58,16 +33,16 @@ case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String
            | "checksum-algorithm" : "$checksumAlgorithm"
            | },
            |""".stripMargin
+    } else {
+      s"""{
+         | "url" : "$accessUrl"
+         | "filepath" : "$filepath"
+         | },
+         |""".stripMargin
     }
-
-    s"""{
-       | "url" : "$accessUrl"
-       | "filepath" : "$filepath"
-       | },
-       |""".stripMargin
   }
 
-  def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
+  private def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
     //write a json file that looks like:
     // [
     //  {
@@ -85,7 +60,8 @@ case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String
     // ]
     var jsonString: String = "[\n"
     for ((accessUrl -> downloadDestination) <- accessUrlToDownloadDest) {
-      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).escapedChecksum, GetmChecksum(hashes, accessUrl).getmAlgorithm)
+      val hashes : Option[Map[String,String]] = Option(Map("" -> ""))
+      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).value, GetmChecksum(hashes, accessUrl).getmAlgorithm)
     }
     jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
     jsonString += "]"
@@ -93,31 +69,19 @@ case class BulkAccessUrlDownloader(urlToDownlaodLocation : Map[AccessUrl, String
   }
 
   def runGetm: IO[GetmResult] = {
-    generateDownloadScript match {
-      case Invalid(errors) =>
-        IO.raiseError(AggregatedMessageException("Error generating access URL download script", errors.toList))
-      case Valid(script) => IO {
-        val copyCommand = Seq("bash", "-c", script)
-        val copyProcess = Process(copyCommand)
-
-        val stderr = new StringBuilder()
-        val errorCapture: String => Unit = { s => stderr.append(s); () }
-
-        // As of `getm` version 0.0.4 the contents of stdout do not appear to be interesting (only a progress bar
-        // with no option to suppress it), so ignore stdout for now. If stdout becomes interesting in future versions
-        // of `getm` it can be captured just like stderr is being captured here.
-        val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
-
-        GetmResult(returnCode, stderr.toString().trim())
-      }
-    }
+    val script = generateBulkDownloadScript
+    val copyCommand : Seq[String] = Seq("bash", "-c", script.get)
+    val copyProcess = Process(copyCommand)
+    val stderr = new StringBuilder()
+    val errorCapture: String => Unit = { s => stderr.append(s); () }
+    val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
+    IO(GetmResult(returnCode, stderr.toString().trim()))
   }
 
   override def download: IO[DownloadResult] = {
     // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's
     // workspace bucket, but that bucket may have visibility different than the data referenced by the signed URL.
-    val masked = accessUrl.url.maskSensitiveUri
-    logger.info(s"Attempting to download data to '$downloadLoc' from access URL '$masked'.")
+    logger.info(s"Attempting to download data")
 
     runGetm map toDownloadResult
   }
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
index 76fb1e426a9..55e5ff4134a 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
@@ -7,7 +7,7 @@ import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 trait DownloaderFactory {
   def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader]
 
-  def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap : Map[AccessUrl, String], hashes: Hashes) : IO[Downloader]
+  def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap : Map[AccessUrl, String]) : IO[Downloader]
 
   def buildGcsUriDownloader(gcsPath: String,
                             serviceAccountJsonOption: Option[String],
diff --git a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
deleted file mode 100644
index 6fc9392fd95..00000000000
--- a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.194f43e1-319b-402d-9c1d-ff704a1db2a2.log
+++ /dev/null
@@ -1,585 +0,0 @@
-2023-07-27 11:43:30,007 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Parsing workflow as WDL draft-2
-2023-07-27 11:43:30,016 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Call-to-Backend assignments: wf_hello.hello -> TES
-2023-07-27 11:43:30,407 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:45:40,558 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Parsing workflow as WDL draft-2
-2023-07-27 11:45:40,567 INFO  - MaterializeWorkflowDescriptorActor [UUID(194f43e1)]: Call-to-Backend assignments: wf_hello.hello -> TES
-2023-07-27 11:45:41,848 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:46:42,262 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 2. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:47:42,795 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 3. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:48:43,290 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 4. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:49:43,688 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 5. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:50:44,159 INFO  - WorkflowActor-194f43e1-319b-402d-9c1d-ff704a1db2a2 [UUID(194f43e1)]: Initialization failed on attempt 6. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
diff --git a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log b/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log
deleted file mode 100644
index 52d8225d72b..00000000000
--- a/https:/lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs/workflow.d81c2541-2cc2-4554-9aee-cb291c63ad7c.log
+++ /dev/null
@@ -1,670 +0,0 @@
-2023-07-27 11:41:05,061 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
-2023-07-27 11:41:05,559 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
-2023-07-27 11:41:07,007 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:42:56,106 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
-2023-07-27 11:42:56,535 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
-2023-07-27 11:42:57,853 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:45:39,659 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Parsing workflow as WDL draft-2
-2023-07-27 11:45:40,103 INFO  - MaterializeWorkflowDescriptorActor [UUID(d81c2541)]: Call-to-Backend assignments: wf_hello.hello -> TES
-2023-07-27 11:45:41,493 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 1. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:46:41,908 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 2. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:47:42,407 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 3. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:48:42,874 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 4. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:49:43,277 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 5. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted
-2023-07-27 11:50:43,751 INFO  - WorkflowActor-d81c2541-2cc2-4554-9aee-cb291c63ad7c [UUID(d81c2541)]: Initialization failed on attempt 6. Will retry up to 16 times. Next retry is in 1 minute
-cromwell.core.CromwellAggregatedException: Initialization Failure:
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:353)
-	at cromwell.engine.workflow.WorkflowActor$$anonfun$3.applyOrElse(WorkflowActor.scala:336)
-	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
-	at akka.actor.FSM.processEvent(FSM.scala:707)
-	at akka.actor.FSM.processEvent$(FSM.scala:704)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowActor.scala:222)
-	at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
-	at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
-	at cromwell.engine.workflow.WorkflowActor.processEvent(WorkflowActor.scala:222)
-	at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
-	at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
-	at akka.actor.Actor.aroundReceive(Actor.scala:539)
-	at akka.actor.Actor.aroundReceive$(Actor.scala:537)
-	at cromwell.engine.workflow.WorkflowActor.akka$actor$Timers$$super$aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.Timers.aroundReceive(Timers.scala:55)
-	at akka.actor.Timers.aroundReceive$(Timers.scala:40)
-	at cromwell.engine.workflow.WorkflowActor.aroundReceive(WorkflowActor.scala:222)
-	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
-	at akka.actor.ActorCell.invoke(ActorCell.scala:583)
-	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
-	at akka.dispatch.Mailbox.run(Mailbox.scala:229)
-	at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
-	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
-	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
-	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
-	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
-	Suppressed: bio.terra.workspace.client.ApiException: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
-<html><head><script src="https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js" tcellappid="FCNonprod-NaVu9" tcellapikey="AQQBBAFLGLOxL7VE9IF9ESlLvCxD5Ykr_7xkQKq_rgn_P58IWjOhOzIh6p3aI4pTWaprlUw" tcellbaseurl="https://us.agent.tcell.insight.rapid7.com/api/v1"></script>
-<title>401 Unauthorized</title>
-</head><body>
-<h1>Unauthorized</h1>
-<p>This server could not verify that you
-are authorized to access the document
-requested.  Either you supplied the wrong
-credentials (e.g., bad password), or your
-browser doesn't understand how to supply
-the credentials required.</p>
-</body></html>
-
-		at bio.terra.workspace.client.ApiClient.invokeAPI(ApiClient.java:752)
-		at bio.terra.workspace.api.ControlledAzureResourceApi.createAzureStorageContainerSasToken(ControlledAzureResourceApi.java:434)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$3(BlobFileSystemManager.scala:213)
-		at scala.util.Try$.apply(Try.scala:210)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.$anonfun$generateBlobSasToken$2(BlobFileSystemManager.scala:214)
-		at scala.util.Success.flatMap(Try.scala:258)
-		at cromwell.filesystems.blob.WSMBlobSasTokenGenerator.generateBlobSasToken(BlobFileSystemManager.scala:204)
-		at cromwell.filesystems.blob.BlobFileSystemManager.retrieveFilesystem(BlobFileSystemManager.scala:84)
-		at cromwell.filesystems.blob.BlobPath.findNioPath(BlobPathBuilder.scala:109)
-		at cromwell.filesystems.blob.BlobPath.nioPath(BlobPathBuilder.scala:99)
-		at cromwell.core.path.Path.nioPathPrivate(PathBuilder.scala:135)
-		at cromwell.core.path.Path.nioPathPrivate$(PathBuilder.scala:135)
-		at cromwell.filesystems.blob.BlobPath.nioPathPrivate(BlobPathBuilder.scala:98)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath(NioPathMethods.scala:31)
-		at cromwell.core.path.NioPathMethods.toAbsolutePath$(NioPathMethods.scala:31)
-		at cromwell.filesystems.blob.BlobPath.toAbsolutePath(BlobPathBuilder.scala:98)
-		at cromwell.backend.io.WorkflowPaths.executionRoot(WorkflowPaths.scala:35)
-		at cromwell.backend.io.WorkflowPaths.executionRoot$(WorkflowPaths.scala:35)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.executionRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot(WorkflowPaths.scala:46)
-		at cromwell.backend.io.WorkflowPaths.workflowRoot$(WorkflowPaths.scala:46)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot$lzycompute(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesWorkflowPaths.workflowRoot(TesWorkflowPaths.scala:9)
-		at cromwell.backend.impl.tes.TesInitializationActor.$anonfun$beforeAll$1(TesInitializationActor.scala:91)
-		at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
-		at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
-		at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
-		... 4 common frames omitted

From 294f0c374fb8d29515ddd937960cdee957dfc9b7 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 1 Sep 2023 16:03:31 -0400
Subject: [PATCH 14/36] somethin that kinda works

---
 .../drs/localizer/DrsLocalizerMain.scala      | 42 ++++-------
 .../downloaders/BulkAccessUrlDownloader.scala | 50 ++++++-------
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 30 ++++++++
 ...o template_ Cromwell DRS Localizer.run.xml |  2 +-
 src/ci/resources/tes_application.conf         | 75 ++++++++++---------
 5 files changed, 105 insertions(+), 94 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 37e99ed4d32..c471156ef7e 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -7,6 +7,7 @@ import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
+import drs.localizer.downloaders.AccessUrlDownloader
 import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
@@ -44,10 +45,9 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
 
   val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory {
-    override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-      val downloader = AccessUrlDownloader(accessUrl, downloadLoc, hashes)
-      IO.pure(downloader)
-    }
+    override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] =
+      IO.pure(AccessUrlDownloader(accessUrl, downloadLoc, hashes))
+
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
       IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
 
@@ -61,27 +61,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     IO.pure(ExitCode.Error)
   }
 
-  /*
-  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
-    commandLineArguments.manifestPath match {
-      case Some(manifestPath) =>
-        val manifestFile = new File(manifestPath)
-        val csvParser = CSVParser.parse(manifestFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
-        val exitCodes: IO[List[ExitCode]] = csvParser.asScala.map(record => {
-          val drsObject = record.get(0)
-          val containerPath = record.get(1)
-          localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
-        }).toList.sequence
-        exitCodes.map(_.find(_ != ExitCode.Success).getOrElse(ExitCode.Success))
-      case None =>
-        val drsObject = commandLineArguments.drsObject.get
-        val containerPath = commandLineArguments.containerPath.get
-        localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
-    }
-  }
-  */
-
-  def getDrsPathResolver(drsCredentials: DrsCredentials): IO[DrsLocalizerDrsPathResolver] = {
+  private def getDrsPathResolver(drsCredentials: DrsCredentials): IO[DrsLocalizerDrsPathResolver] = {
     IO {
       val drsConfig = DrsConfig.fromEnv(sys.env)
       logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects")
@@ -113,8 +93,8 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
               return IO(ExitCode.Error) //Could not resolve DRS urls. Fail before attempting to download anything
           }
 
-          val accessUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.HTTPS)
-          val googleUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == URIType.GCS)
+          val accessUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == Try(URIType.HTTPS))
+          val googleUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == Try(URIType.GCS))
 
           val bulkAccessResult : IO[ExitCode] = bulkDownloadResolvedAccessUrls(accessUrls, drsCredentials)
           val googleResult : List[IO[DownloadResult]] = bulkDownloadResolvedGoogleUrls(googleUrls, commandLineArguments.googleRequesterPaysProject)
@@ -129,13 +109,19 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       case None =>
         val drsObject = commandLineArguments.drsObject.get
         val containerPath = commandLineArguments.containerPath.get
+        val resolvedURI = resolveDrsUrl(resolver, drsObject)
+        val map : Map[DrsResolverResponse, String] = Map((resolvedURI.unsafeRunSync() -> containerPath))
+        bulkDownloadResolvedAccessUrls(map, drsCredentials)
         localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
     }
   }
 
-  def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
+  private def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
     val accessUrlMap = resolvedUrlToOutputDestination.map( pair => pair._1.accessUrl.get -> pair._2)
     val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(accessUrlMap)
+    val unwrappedDownloader = downloader.unsafeRunSync()
+    val downloadResult = unwrappedDownloader.download
+    downloadResult.toString()
     downloader.flatMap(_.download)
     IO(ExitCode.Success) //tood
   }
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index fe6687d4b8c..a01034d2db9 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -10,34 +10,31 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path, Paths}
 import scala.sys.process.{Process, ProcessLogger}
 import scala.util.Try
-import scala.util.matching.Regex
-
-case class GetmResult(returnCode: Int, stderr: String)
 
 case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String]) extends Downloader with StrictLogging {
 
   private def generateBulkDownloadScript: Try[String] = {
-    val manifestPath = generateJsonManifest(urlToDownloadLocation)
+    val manifestPath = generateJsonManifest(urlToDownloadLocation).get
     val downloadLoc = "getmDL"
-    val checksumArgs = "pleaseCompile"
-    generateJsonManifest(urlToDownloadLocation).map(mainfestPath =>  s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --manifest $manifestPath'""")
+    //val checksumArgs = "pleaseCompile"
+    generateJsonManifest(urlToDownloadLocation).map(_ =>  s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm --manifest '$manifestPath'""")
   }
 
   private def toJsonString(accessUrl: String, filepath: String, checksum: ErrorOr[String], checksumAlgorithm: String): String = {
     //NB: trailing comma is being removed in generateJsonManifest
     if (checksum.isValid) {
-        s"""{
-           | "url" : "$accessUrl"
-           | "filepath" : "$filepath"
-           | "checksum" : "$checksum"
-           | "checksum-algorithm" : "$checksumAlgorithm"
-           | },
+        s"""  {
+           |    "url" : "$accessUrl",
+           |    "filepath" : "$filepath",
+           |    "checksum" : "$checksum",
+           |    "checksum-algorithm" : "$checksumAlgorithm"
+           |  },
            |""".stripMargin
     } else {
-      s"""{
-         | "url" : "$accessUrl"
-         | "filepath" : "$filepath"
-         | },
+      s"""  {
+         |    "url" : "$accessUrl",
+         |    "filepath" : "$filepath"
+         |  },
          |""".stripMargin
     }
   }
@@ -46,15 +43,15 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
     //write a json file that looks like:
     // [
     //  {
-    //     "url" : "www.123.com"
-    //     "filepath" : "path/to/where/123/should/be/downloaded"
-    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf"
+    //     "url" : "www.123.com",
+    //     "filepath" : "path/to/where/123/should/be/downloaded",
+    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf",
     //     "checksum-algorithm" : "md5"
     //  },
     //  {
     //     "url" : "www.567.com"
-    //     "filepath" : "path/to/where/567/should/be/downloaded"
-    //     "checksum" : "asdasdasfsdfsdfasdsdfasd"
+    //     "filepath" : "path/to/where/567/should/be/downloaded",
+    //     "checksum" : "asdasdasfsdfsdfasdsdfasd",
     //     "checksum-algorithm" : "md5"
     //  }
     // ]
@@ -64,17 +61,19 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
       jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).value, GetmChecksum(hashes, accessUrl).getmAlgorithm)
     }
     jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
-    jsonString += "]"
+    jsonString += "\n]"
     Try(Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8)))
   }
 
   def runGetm: IO[GetmResult] = {
     val script = generateBulkDownloadScript
     val copyCommand : Seq[String] = Seq("bash", "-c", script.get)
+    logger.info(script.get)
     val copyProcess = Process(copyCommand)
     val stderr = new StringBuilder()
     val errorCapture: String => Unit = { s => stderr.append(s); () }
     val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
+    logger.info(stderr.toString().trim())
     IO(GetmResult(returnCode, stderr.toString().trim()))
   }
 
@@ -116,10 +115,3 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
     }
   }
 }
-
-object AccessUrlDownloader {
-  type Hashes = Option[Map[String, String]]
-
-  val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r
-  val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r
-}
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 66799fcc099..0602a1dd73c 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -99,6 +99,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     drsLocalizer.resolveAndDownloadWithRetries(
@@ -134,6 +139,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     assertThrows[Throwable] {
@@ -172,6 +182,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     assertThrows[Throwable] {
@@ -208,6 +223,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
         gcsUriDownloader
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     drsLocalizer.resolveAndDownloadWithRetries(
@@ -241,6 +261,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
         gcsUriDownloader
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     assertThrows[Throwable] {
@@ -276,6 +301,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
         // This test path should never ask for the GCS URI downloader.
         throw new RuntimeException("test failure")
       }
+
+      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
+      }
     }
 
     assertThrows[Throwable] {
diff --git a/runConfigurations/Repo template_ Cromwell DRS Localizer.run.xml b/runConfigurations/Repo template_ Cromwell DRS Localizer.run.xml
index 91b36d9277e..12f01e0179e 100644
--- a/runConfigurations/Repo template_ Cromwell DRS Localizer.run.xml	
+++ b/runConfigurations/Repo template_ Cromwell DRS Localizer.run.xml	
@@ -5,7 +5,7 @@
     </envs>
     <option name="MAIN_CLASS_NAME" value="drs.localizer.DrsLocalizerMain" />
     <module name="cromwell-drs-localizer" />
-    <option name="PROGRAM_PARAMETERS" value="--access-token-strategy azure drs://jade.datarepo-dev.broadinstitute.org/v1_f3c32ddd-20eb-4033-9716-46866521a3a4_8d651e95-38ce-4bc8-a8a4-8a4cd9513db1 $USER_HOME$/Downloads/NA12878_PLUMBING_wgs.g.vcf.gz.tbi --identity-client-id identity" />
+    <option name="PROGRAM_PARAMETERS" value="--access-token-strategy google drs://drs.anv0:v1_e2151834-13cd-4156-9ea2-168a1b7abf60_0761203d-d2a1-448e-8f71-9f81d80ddd9d $USER_HOME$/Downloads/test/plumbing" />
     <method v="2">
       <option name="Make" enabled="true" />
     </method>
diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf
index 5eb812a52d4..9a8642871ae 100644
--- a/src/ci/resources/tes_application.conf
+++ b/src/ci/resources/tes_application.conf
@@ -2,40 +2,36 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
 filesystems {
-  blob {
-    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-    global {
-      # One BFSM is shared across all BlobPathBuilders
-      class = "cromwell.filesystems.blob.BlobFileSystemManager"
-      config {
-        container: "cromwell"
-        endpoint: "https://<storage-account>.blob.core.windows.net"
-        subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
-        # WSM config is needed for accessing WSM-managed blob containers
-        # created in Terra workspaces.
-        workspace-manager {
-          url: "https://workspace.dsde-dev.broadinstitute.org"
-          workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          b2cToken: "Zardoz"
+    blob {
+        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+        global {
+            class = "cromwell.filesystems.blob.BlobFileSystemManager"
+            config {
+                workspace-manager: {
+                    url: "https://workspace.dsde-dev.broadinstitute.org"
+                    workspace-id: "d4564046-bbba-495c-afec-14f7d3a8283a"
+                    container-resource-id: "9e7bdba5-047e-430c-810d-c7cb6645c155"
+                    b2cToken: "eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlCUHk1MzZwaHZmT2NlQUhjdzdOMG5OVmxSSDVwdkFyOWE2S3ZKbnFPQjQycVZkYzVOUDlJc2Jndk1iakNSZ2YycWRuaUhhMVpZOTlfdThMUTNFWGUzRG80SEhBQkctV2VEVTVmeHZOT1lRNjdNbS16cGRYN2JnOXl5WU5UcTJ3T2lObjNuMmh2bTBOdHBXR2VDOEJJekxZc0hLTzljU3RiS3NWUU5HVGNYa2FDZ1lLQWZZU0FSQVNGUUhzdllsczZoNTVrendUdlBMRTVPd2JMNktueVEwMTgzIiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTI3Mjc2NjEsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MjczMTI2MSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkyNzI3NjYxfQ.OyIZkNqiGQ_cIih0RYSBbin-BaG7B605MTH7RUIsZQOsLrkD1NR0U5q6tK0BO3A-NqwV4V26suJISGIHF4MlgdFji4p3499p-m7STZ8isQx3644n2jzpvjqaHgVxfYIofEMcDezOo7xZdCViq0UhiC3ajiWTgQmzy-q31L50yqz28ZiwAheTSNbcNyZtVE8ogImpYoem_3qfw-iJu3Sb0u_6OUgsvuiRoYbV-Sh2a4vIeyWlLI8Z6cW5Mrk-xzWSpp7EgnaaxYNye-0PLq5ziQV5DkxZ8XLupnn511wVJrYWkqJ91vEhuXfKLn8ZYnc7iUuJNxoJESRPj0PIit3Ysg"
+                }
+                container: "sc-d4564046-bbba-495c-afec-14f7d3a8283a"
+                endpoint: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net"
+            }
         }
-      }
     }
-  }
 }
 
 engine {
-  filesystems {
-    local {
-      enabled: true
-    }
-    http {
-      enabled: true
-    }
-    blob {
-      enabled: false
+    filesystems {
+        local {
+            enabled: true
+        }
+        http {
+            enabled: true
+        }
+        blob {
+            enabled: true
+        }
     }
-  }
 }
 
 
@@ -46,26 +42,28 @@ backend {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
         # Use for running on blob storage
-        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
-        root = "cromwell-executions"
+        root = "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/"
         dockerRoot = "/cromwell-executions"
-        endpoint = "http://127.0.0.1:9000/v1/tasks"
+        endpoint = "https://lzd98e904c9c1cee247af683bc764eb1b22f3912f8e7839afe.servicebus.windows.net/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/tes/v1/tasks"
         concurrent-job-limit = 1000
-
+        workflow-execution-identity = "pet-2676571657071603a9eab"
+        bearer-token= "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlCUHk1MzZwaHZmT2NlQUhjdzdOMG5OVmxSSDVwdkFyOWE2S3ZKbnFPQjQycVZkYzVOUDlJc2Jndk1iakNSZ2YycWRuaUhhMVpZOTlfdThMUTNFWGUzRG80SEhBQkctV2VEVTVmeHZOT1lRNjdNbS16cGRYN2JnOXl5WU5UcTJ3T2lObjNuMmh2bTBOdHBXR2VDOEJJekxZc0hLTzljU3RiS3NWUU5HVGNYa2FDZ1lLQWZZU0FSQVNGUUhzdllsczZoNTVrendUdlBMRTVPd2JMNktueVEwMTgzIiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTI3Mjc2NjEsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MjczMTI2MSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkyNzI3NjYxfQ.OyIZkNqiGQ_cIih0RYSBbin-BaG7B605MTH7RUIsZQOsLrkD1NR0U5q6tK0BO3A-NqwV4V26suJISGIHF4MlgdFji4p3499p-m7STZ8isQx3644n2jzpvjqaHgVxfYIofEMcDezOo7xZdCViq0UhiC3ajiWTgQmzy-q31L50yqz28ZiwAheTSNbcNyZtVE8ogImpYoem_3qfw-iJu3Sb0u_6OUgsvuiRoYbV-Sh2a4vIeyWlLI8Z6cW5Mrk-xzWSpp7EgnaaxYNye-0PLq5ziQV5DkxZ8XLupnn511wVJrYWkqJ91vEhuXfKLn8ZYnc7iUuJNxoJESRPj0PIit3Ysg"
         # Override default polling to make it faster for speedy tests
         poll-backoff {
-          min: "1 seconds"
+          min: "10 seconds"
           max: "5 minutes"
           multiplier: 1.1
           randomization-factor: 0.5
         }
-        
+
+        use_tes_11_preview_backend_parameters = true
+
         filesystems {
           blob {
-            enabled: false
+            enabled: true
           }
           local {
-            enabled: true
+            enabled: false
           }
           http {
             enabled: true
@@ -78,3 +76,8 @@ backend {
     }
   }
 }
+
+workflow-options {
+  # Directory where to write per workflow logs
+  workflow-log-dir: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs"
+}
\ No newline at end of file

From afe84dafb6f32931ceff4cea92a8741401dca2ab Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 5 Sep 2023 15:15:56 -0400
Subject: [PATCH 15/36] working test

---
 .../downloaders/BulkAccessUrlDownloader.scala |  2 +-
 .../BulkAccessUrlDownloaderSpec.scala         | 45 +++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index a01034d2db9..a2896f77a1d 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -39,7 +39,7 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
     }
   }
 
-  private def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
+  def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
     //write a json file that looks like:
     // [
     //  {
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
new file mode 100644
index 00000000000..7f39f86ea15
--- /dev/null
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -0,0 +1,45 @@
+package drs.localizer.downloaders
+
+import cloud.nio.impl.drs.AccessUrl
+import common.assertion.CromwellTimeoutSpec
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.file.Path
+import scala.util.Try
+
+
+class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
+
+  it should "correctly parse a collection of Access Urls into a manifest.json" in {
+
+    val fakeAccessURLToDownloadLocationMap : Map[AccessUrl, String] = Map(
+      (AccessUrl("https://my.fake/url123", None) -> "./test1.out"),
+      (AccessUrl("https://my.fake/url1234", None) -> "./test2.out")
+    )
+
+    val expected : String =
+      s"""[
+         |  {
+         |    "url" : "https://my.fake/url123",
+         |    "filepath" : "./test1.out",
+         |    "checksum" : "Valid()",
+         |    "checksum-algorithm" : "null"
+         |  },
+         |  {
+         |    "url" : "https://my.fake/url1234",
+         |    "filepath" : "./test2.out",
+         |    "checksum" : "Valid()",
+         |    "checksum-algorithm" : "null"
+         |  }
+         |]""".stripMargin
+
+    val downloader = BulkAccessUrlDownloader(fakeAccessURLToDownloadLocationMap)
+
+    val filepath : Try[Path] = downloader.generateJsonManifest(fakeAccessURLToDownloadLocationMap)
+    val source = scala.io.Source.fromFile(filepath.get.toString)
+    val lines = try source.mkString finally source.close()
+
+    lines shouldBe expected
+  }
+}

From 0bd1a8da39b7f37a22565d2ba4b616b6bdac2823 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 7 Sep 2023 16:50:02 -0400
Subject: [PATCH 16/36] lots of good cleanup

---
 .../drs/localizer/DrsLocalizerMain.scala      | 219 +++++++-----------
 .../downloaders/AccessUrlDownloader.scala     |  91 --------
 .../downloaders/BulkAccessUrlDownloader.scala | 130 ++++++-----
 .../downloaders/DownloaderFactory.scala       |   7 +-
 .../localizer/downloaders/GetmChecksum.scala  |   2 +-
 .../drs/localizer/DrsLocalizerMainSpec.scala  |   6 +-
 .../downloaders/AccessUrlDownloaderSpec.scala |   5 +-
 7 files changed, 168 insertions(+), 292 deletions(-)
 delete mode 100644 cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index c471156ef7e..d8eaab4809b 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -2,25 +2,24 @@ package drs.localizer
 
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO, IOApp}
-import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition}
+import cats.implicits.toTraverseOps
 import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
-import drs.localizer.downloaders.AccessUrlDownloader
-import drs.localizer.downloaders.AccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
 
-import java.io.{File}
+import java.io.File
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
-import drs.localizer.URIType.{URIType}
+import drs.localizer.URIType.URIType
 
 import scala.util.{Failure, Try}
-
+case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
+case class ResolvedDrsUrlPendingDownload(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
 
   override def run(args: List[String]): IO[ExitCode] = {
@@ -45,14 +44,11 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
 
   val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory {
-    override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] =
-      IO.pure(AccessUrlDownloader(accessUrl, downloadLoc, hashes))
-
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
       IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
 
-    override def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap: Map[AccessUrl, String]): IO[Downloader] = {
-      IO.pure(BulkAccessUrlDownloader(urlToDownloadDestinationMap))
+    override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrlPendingDownload]): IO[Downloader] = {
+      IO.pure(BulkAccessUrlDownloader(urlsToDownload))
     }
   }
 
@@ -69,129 +65,115 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
 
-  private[localizer] def toUriType(drsResponse: DrsResolverResponse): Try[URIType] = {
-    (drsResponse.accessUrl, drsResponse.gsUri) match {
-      case (Some(_), _) =>
-        Try(URIType.HTTPS)
+  private[localizer] def toUriType(accessUrl: Option[AccessUrl], gsUri : Option[String]): URIType = {
+    (accessUrl, gsUri) match {
       case (_, Some(_)) =>
-        Try(URIType.GCS)
-      case _ =>
-        Failure(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
+        URIType.GCS
+      case (Some(_), _) =>
+        URIType.HTTPS
+      case (_,_) =>
+        URIType.UNKNOWN
     }
   }
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
-    //todo: just fail here if this can't be gotten
-    val resolver = getDrsPathResolver(drsCredentials).unsafeRunSync()
-    commandLineArguments.manifestPath match {
-      case Some(manifestPath) => {
-          val resolvedUris: Try[Map[DrsResolverResponse, String]] = for {
-            resolvedDrsUris: Map[DrsResolverResponse, String] <- resolveCSVManifest(manifestPath, resolver)
-            if !resolvedDrsUris.isEmpty
-          } yield resolvedDrsUris
-
-          if(resolvedUris.isFailure) {
-              return IO(ExitCode.Error) //Could not resolve DRS urls. Fail before attempting to download anything
-          }
-
-          val accessUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == Try(URIType.HTTPS))
-          val googleUrls = resolvedUris.get.filter(drsResponse => toUriType(drsResponse._1) == Try(URIType.GCS))
-
-          val bulkAccessResult : IO[ExitCode] = bulkDownloadResolvedAccessUrls(accessUrls, drsCredentials)
-          val googleResult : List[IO[DownloadResult]] = bulkDownloadResolvedGoogleUrls(googleUrls, commandLineArguments.googleRequesterPaysProject)
-
-        if(bulkAccessResult.equals(true) && googleResult.equals(true)) {
-          IO(ExitCode.Success)
-        } else {
-          IO(ExitCode.Error)
-        }
-
+    IO {
+      // Get a single resolver to resolve URLs with
+      val resolver = getDrsPathResolver(drsCredentials).unsafeRunSync()
+
+      val exitCode : ExitCode = commandLineArguments.manifestPath match {
+        case Some(manifestPath) => {
+          val unresolvedUris: IO[List[UnresolvedDrsUrl]] = loadCSVManifest(manifestPath)
+          val resolvedUris: IO[List[ResolvedDrsUrlPendingDownload]] = unresolvedUris.flatMap(uris => uris.map(toResolve => resolveDrsUrl(resolver, toResolve)).traverse(identity))
+
+          // Resolve all the URLs, sorting them into buckets by their required downloader type
+          val resolvedUrisByType: Map[URIType, List[ResolvedDrsUrlPendingDownload]] = resolvedUris.map { resolvedList =>
+            Map(
+              URIType.GCS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.GCS),
+              URIType.HTTPS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.HTTPS),
+              URIType.UNKNOWN -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.UNKNOWN)
+            )
+          }.unsafeRunSync()
+
+          val googleDownload : IO[List[DownloadResult]] = resolvedUrisByType.get(URIType.GCS).map(googleUris => bulkDownloadResolvedGoogleUrls(googleUris, commandLineArguments.googleRequesterPaysProject)).getOrElse(List())
+          val bulkAccessDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.HTTPS).map(resolvedUris => bulkDownloadResolvedAccessUrls(resolvedUris)).getOrElse(IO(DownloadSuccess))
+          resolvedUrisByType.get(URIType.UNKNOWN).map(_ => logger.error("Unrecognized URI format returned from resolver"))
+
+          val bulkAccessResult = bulkAccessDownload.unsafeRunSync()
+          val googleResults = googleDownload.unsafeRunSync()
+          yield if (googleResults.find(res => res != DownloadSuccess).isEmpty && bulkAccessResult.exitCode == ExitCode.Success) ExitCode.Success else ExitCode.Error
         }
-      case None =>
-        val drsObject = commandLineArguments.drsObject.get
-        val containerPath = commandLineArguments.containerPath.get
-        val resolvedURI = resolveDrsUrl(resolver, drsObject)
-        val map : Map[DrsResolverResponse, String] = Map((resolvedURI.unsafeRunSync() -> containerPath))
-        bulkDownloadResolvedAccessUrls(map, drsCredentials)
-        localizeFile(commandLineArguments, drsCredentials, drsObject, containerPath)
+        case None =>
+          val drsObject = commandLineArguments.drsObject.get
+          val containerPath = commandLineArguments.containerPath.get
+          val resolvedURI = resolveDrsUrl(resolver, drsObject)
+          val map: Map[DrsResolverResponse, String] = Map((resolvedURI.unsafeRunSync() -> containerPath))
+          bulkDownloadResolvedAccessUrls(map, drsCredentials)
+        //TODO: Also handle the Google download case here.
+      }
     }
   }
 
-  private def bulkDownloadResolvedAccessUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], drsCredentials: DrsCredentials): IO[ExitCode] = {
-    val accessUrlMap = resolvedUrlToOutputDestination.map( pair => pair._1.accessUrl.get -> pair._2)
-    val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(accessUrlMap)
-    val unwrappedDownloader = downloader.unsafeRunSync()
-    val downloadResult = unwrappedDownloader.download
-    downloadResult.toString()
-    downloader.flatMap(_.download)
-    IO(ExitCode.Success) //tood
+  private def bulkDownloadResolvedAccessUrls(resolvedUrls : List[ResolvedDrsUrlPendingDownload]): IO[DownloadResult] = {
+    IO[DownloadResult] {
+      if(resolvedUrls.isEmpty) return IO(DownloadSuccess)
+      val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls).unsafeRunSync()
+      downloader.download.unsafeRunSync()
+    }
   }
 
-  def bulkDownloadResolvedGoogleUrls(resolvedUrlToOutputDestination : Map[DrsResolverResponse, String], googleRequesterPaysProject : Option[String]) : List[IO[DownloadResult]] = {
-    resolvedUrlToOutputDestination.map(pair => {
-      val drsResponse : DrsResolverResponse = pair._1
-      val containerPath : String = pair._2
-      val serviceAccountJsonOption = drsResponse.googleServiceAccount.map(_.data.spaces2)
+  def bulkDownloadResolvedGoogleUrls(resolvedUrls : List[ResolvedDrsUrlPendingDownload], googleRequesterPaysProject : Option[String]) : IO[List[DownloadResult]] = {
+    resolvedUrls.map(url => {
+      val serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2)
       val downloader = defaultDownloaderFactory.buildGcsUriDownloader(
-        gcsPath = drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
+        gcsPath = url.drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
         serviceAccountJsonOption = serviceAccountJsonOption,
-        downloadLoc = containerPath,
+        downloadLoc = url.downloadDestinationPath,
         requesterPaysProjectOption = googleRequesterPaysProject)
       downloader.flatMap(_.download)
-    }).toList
+    }).traverse(identity)
   }
 
-  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: String): IO[DrsResolverResponse] = {
-    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-    resolverObject.resolveDrs(drsUrlToResolve, fields)
+  //NB: This method calls a synchronous HTTP request
+  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrlPendingDownload] = {
+    IO {
+      val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+      //Insert retry logic here.
+      val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
+      ResolvedDrsUrlPendingDownload(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
+    }
   }
   // resolve all the URIs once.
   def resolveCSVManifest(csvManifestPath: String, resolver: DrsLocalizerDrsPathResolver) : Try[Map[DrsResolverResponse, String]] = {
-    val drsUriToDownloadLocationMap: Map[String, String] = loadCSVAsMap(csvManifestPath).get
-    val resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.map(value => (resolveDrsUrl(resolver, value._1).unsafeRunSync(), value._2))
+    val drsUriToDownloadLocationMap: Try[Map[String, String]] = loadCSVAsMap(csvManifestPath)
+    if (drsUriToDownloadLocationMap.isFailure) return Failure[Map[DrsResolverResponse, String]](new RuntimeException("Could not parse CSV manifest"))
+    val resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.get.map(value => (resolveDrsUrl(resolver, value._1).unsafeRunSync(), value._2))
     Try(resolvedUriToDownloadLocationMap)
-    }
-
-  private def localizeFile(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials, drsObject: String, containerPath: String) = {
-    new DrsLocalizerMain(drsObject, containerPath, drsCredentials, commandLineArguments.googleRequesterPaysProject).
-      resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode)
   }
 
-  // Overall plan for bulk DRS downloads:
-  // 1) Open the CSV provided to Cromwell Drs Localizer as a map of DRS URI -> Destination Filepath
-  // 2) Resolve each DRS URI in the map. This will yield two maps, because a drs URI can resolve to either
-  // a gs:// URI or an https:// uri, each of which require different downloaders.
-  // 3a) Invoke the GcsUriDownloader (and its underlying gsutil) to download all gs:// URIs in parallel.
-  // 3b) Invoke the AccessUrlDownloader (and its underlying getm) to download all https:// URIs in parallel.
-  //     - getm requires a --manifest argument, which is a .json file that is a map of https:// URI to destination filepath
-  // 4) Report success or failure as granularly as is reasonably possible.
-  private def loadCSVAsMap(csvManifestPath : String) : Try[Map[String, String]] = {
-    for {
-      openFile <- Try(new File(csvManifestPath))
-      csvParser <- Try(CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT))
-      map <- Try(csvParser.getRecords.asScala.map(record => (record.get(0), record.get(1))).toMap)
-      if !map.isEmpty //An empty map should be considered an error - we're expecting at least one k/v pair in the manifest.
-    } yield map
+  /**
+    * Helper function to read a CSV file as a map from drs URL to requested download destination
+    * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
+    */
+  private def loadCSVManifest(csvManifestPath : String) : IO[List[UnresolvedDrsUrl]] = {
+    IO {
+      val openFile = new File(csvManifestPath)
+      val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
+      val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
+      list
+    }
   }
 }
 
 object URIType extends Enumeration {
   type URIType = Value
-  val GCS, HTTPS = Value
+  val GCS, HTTPS, UNKNOWN = Value
 }
 
 class DrsLocalizerMain(drsUrl: String,
                        downloadLoc: String,
                        drsCredentials: DrsCredentials,
                        requesterPaysProjectIdOption: Option[String]) extends StrictLogging {
-
-  def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
-    IO {
-      val drsConfig = DrsConfig.fromEnv(sys.env)
-      logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects")
-      new DrsLocalizerDrsPathResolver(drsConfig, drsCredentials)
-    }
-  }
-
+  /*
   def resolveAndDownloadWithRetries(downloadRetries: Int,
                                     checksumRetries: Int,
                                     downloaderFactory: DownloaderFactory,
@@ -209,7 +191,8 @@ class DrsLocalizerMain(drsUrl: String,
         IO.raiseError(new RuntimeException(s"Exhausted $checksumRetries checksum retries to resolve, download and checksum $drsUrl", t))
       }
     }
-
+*/
+    /*
     def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = {
       t match {
         case _: FatalRetryDisposition =>
@@ -222,7 +205,8 @@ class DrsLocalizerMain(drsUrl: String,
           IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
       }
     }
-
+*/
+      /*
     resolveAndDownload(downloaderFactory).redeemWith({
       maybeRetryForDownloadFailure
     },
@@ -235,34 +219,11 @@ class DrsLocalizerMain(drsUrl: String,
       case ChecksumFailure =>
         maybeRetryForChecksumFailure(new RuntimeException(s"Checksum failure for $drsUrl on checksum retry attempt $checksumAttempt of $checksumRetries"))
       case o => IO.pure(o)
-    })
-  }
+    }*
 
-  private [localizer] def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-    resolve(downloaderFactory) flatMap { _.download }
-  }
-
-  private [localizer] def resolve(downloaderFactory: DownloaderFactory): IO[Downloader] = {
-    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-    for {
-      resolver <- getDrsPathResolver
-      drsResolverResponse <- resolver.resolveDrs(drsUrl, fields)
 
-      // Currently DRS Resolver only supports resolving DRS paths to access URLs or GCS paths.
-      downloader <- (drsResolverResponse.accessUrl, drsResolverResponse.gsUri) match {
-        case (Some(accessUrl), _) =>
-          downloaderFactory.buildAccessUrlDownloader(accessUrl, downloadLoc, drsResolverResponse.hashes)
-        case (_, Some(gcsPath)) =>
-          val serviceAccountJsonOption = drsResolverResponse.googleServiceAccount.map(_.data.spaces2)
-          downloaderFactory.buildGcsUriDownloader(
-            gcsPath = gcsPath,
-            serviceAccountJsonOption = serviceAccountJsonOption,
-            downloadLoc = downloadLoc,
-            requesterPaysProjectOption = requesterPaysProjectIdOption)
-        case _ =>
-          IO.raiseError(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg))
-      }
-    } yield downloader
+    IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
   }
+         */
 }
 
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
deleted file mode 100644
index ae6f2fa4f1e..00000000000
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-package drs.localizer.downloaders
-
-import cats.data.Validated.{Invalid, Valid}
-import cats.effect.{ExitCode, IO}
-import cloud.nio.impl.drs.AccessUrl
-import com.typesafe.scalalogging.StrictLogging
-import common.exception.AggregatedMessageException
-import common.util.StringUtil._
-import common.validation.ErrorOr.ErrorOr
-import drs.localizer.downloaders.AccessUrlDownloader._
-
-import scala.sys.process.{Process, ProcessLogger}
-import scala.util.matching.Regex
-
-case class GetmResult(returnCode: Int, stderr: String)
-
-case class AccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes) extends Downloader with StrictLogging {
-  def generateDownloadScript: ErrorOr[String] = {
-    val signedUrl = accessUrl.url
-    GetmChecksum(hashes, accessUrl).args map { checksumArgs =>
-      s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --filepath '$downloadLoc' '$signedUrl'"""
-    }
-  }
-
-  def runGetm: IO[GetmResult] = {
-    generateDownloadScript match {
-      case Invalid(errors) =>
-        IO.raiseError(AggregatedMessageException("Error generating access URL download script", errors.toList))
-      case Valid(script) => IO {
-        val copyCommand = Seq("bash", "-c", script)
-        val copyProcess = Process(copyCommand)
-
-        val stderr = new StringBuilder()
-        val errorCapture: String => Unit = { s => stderr.append(s); () }
-
-        // As of `getm` version 0.0.4 the contents of stdout do not appear to be interesting (only a progress bar
-        // with no option to suppress it), so ignore stdout for now. If stdout becomes interesting in future versions
-        // of `getm` it can be captured just like stderr is being captured here.
-        val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
-
-        GetmResult(returnCode, stderr.toString().trim())
-      }
-    }
-  }
-
-  override def download: IO[DownloadResult] = {
-    // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's
-    // workspace bucket, but that bucket may have visibility different than the data referenced by the signed URL.
-    val masked = accessUrl.url.maskSensitiveUri
-    logger.info(s"Attempting to download data to '$downloadLoc' from access URL '$masked'.")
-
-    runGetm map toDownloadResult
-  }
-
-  def toDownloadResult(getmResult: GetmResult): DownloadResult = {
-    getmResult match {
-      case GetmResult(0, stderr) if stderr.isEmpty =>
-        DownloadSuccess
-      case GetmResult(0, stderr) =>
-        stderr match {
-          case ChecksumFailureMessage() =>
-            ChecksumFailure
-          case _ =>
-            UnrecognizedRetryableDownloadFailure(ExitCode(0))
-        }
-      case GetmResult(rc, stderr) =>
-        stderr match {
-          case HttpStatusMessage(status) =>
-            Integer.parseInt(status) match {
-              case 408 | 429 =>
-                RecognizedRetryableDownloadFailure(ExitCode(rc))
-              case s if s / 100 == 4 =>
-                FatalDownloadFailure(ExitCode(rc))
-              case s if s / 100 == 5 =>
-                RecognizedRetryableDownloadFailure(ExitCode(rc))
-              case _ =>
-                UnrecognizedRetryableDownloadFailure(ExitCode(rc))
-            }
-          case _ =>
-            UnrecognizedRetryableDownloadFailure(ExitCode(rc))
-        }
-    }
-  }
-}
-
-object AccessUrlDownloader {
-  type Hashes = Option[Map[String, String]]
-
-  val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r
-  val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r
-}
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index a2896f77a1d..e1502c53e0b 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -1,80 +1,85 @@
 package drs.localizer.downloaders
 
 import cats.effect.{ExitCode, IO}
-import cloud.nio.impl.drs.AccessUrl
+import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse}
 import com.typesafe.scalalogging.StrictLogging
-import common.validation.ErrorOr.ErrorOr
-import drs.localizer.downloaders.AccessUrlDownloader._
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path, Paths}
 import scala.sys.process.{Process, ProcessLogger}
-import scala.util.Try
+import scala.util.matching.Regex
+import drs.localizer.ResolvedDrsUrlPendingDownload
+case class GetmResult(returnCode: Int, stderr: String)
 
-case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String]) extends Downloader with StrictLogging {
+/**
+  * Getm is a python tool that is used to download resolved DRS uris quickly and in parallel.
+  * This class builds a getm-manifest.json file that it uses for input, and builds/executes a shell command
+  * to invoke the Getm tool, which is expected to already be installed in the local environment.
+  * @param resolvedUrls
+  */
+case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrlPendingDownload]) extends Downloader with StrictLogging {
+  /**
+    * Write a json manifest to disk that looks like:
+    * // [
+    * //  {
+    * //     "url" : "www.123.com",
+    * //     "filepath" : "path/to/where/123/should/be/downloaded",
+    * //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf",
+    * //     "checksum-algorithm" : "md5"
+    * //  },
+    * //  {
+    * //     "url" : "www.567.com"
+    * //     "filepath" : "path/to/where/567/should/be/downloaded",
+    * //     "checksum" : "asdasdasfsdfsdfasdsdfasd",
+    * //     "checksum-algorithm" : "md5"
+    * //  }
+    * // ]
+    *
+    * @param resolvedUrls
+    * @return Filepath of a getm-manifest.json that Getm can use to download multiple files in parallel.
+    */
+  def generateJsonManifest(resolvedUrls : List[ResolvedDrsUrlPendingDownload]): IO[Path] = {
 
-  private def generateBulkDownloadScript: Try[String] = {
-    val manifestPath = generateJsonManifest(urlToDownloadLocation).get
-    val downloadLoc = "getmDL"
-    //val checksumArgs = "pleaseCompile"
-    generateJsonManifest(urlToDownloadLocation).map(_ =>  s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm --manifest '$manifestPath'""")
-  }
-
-  private def toJsonString(accessUrl: String, filepath: String, checksum: ErrorOr[String], checksumAlgorithm: String): String = {
-    //NB: trailing comma is being removed in generateJsonManifest
-    if (checksum.isValid) {
-        s"""  {
-           |    "url" : "$accessUrl",
-           |    "filepath" : "$filepath",
-           |    "checksum" : "$checksum",
-           |    "checksum-algorithm" : "$checksumAlgorithm"
-           |  },
-           |""".stripMargin
-    } else {
+    def toJsonString(drsResponse: DrsResolverResponse, destinationFilepath: String): String = {
+      //NB: trailing comma is being removed in generateJsonManifest
+      val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse("missing_url")
+      val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
+      val checksumAlgorithm = checksum.getmAlgorithm
       s"""  {
          |    "url" : "$accessUrl",
-         |    "filepath" : "$filepath"
+         |    "filepath" : "$destinationFilepath",
+         |    "checksum" : "$checksum",
+         |    "checksum-algorithm" : "$checksumAlgorithm"
          |  },
          |""".stripMargin
     }
-  }
 
-  def generateJsonManifest(accessUrlToDownloadDest: Map[AccessUrl, String]): Try[Path] = {
-    //write a json file that looks like:
-    // [
-    //  {
-    //     "url" : "www.123.com",
-    //     "filepath" : "path/to/where/123/should/be/downloaded",
-    //     "checksum" : "sdfjndsfjkfsdjsdfkjsdf",
-    //     "checksum-algorithm" : "md5"
-    //  },
-    //  {
-    //     "url" : "www.567.com"
-    //     "filepath" : "path/to/where/567/should/be/downloaded",
-    //     "checksum" : "asdasdasfsdfsdfasdsdfasd",
-    //     "checksum-algorithm" : "md5"
-    //  }
-    // ]
-    var jsonString: String = "[\n"
-    for ((accessUrl -> downloadDestination) <- accessUrlToDownloadDest) {
-      val hashes : Option[Map[String,String]] = Option(Map("" -> ""))
-      jsonString += toJsonString(accessUrl.url, downloadDestination, GetmChecksum(hashes, accessUrl).value, GetmChecksum(hashes, accessUrl).getmAlgorithm)
+    IO {
+      var jsonString: String = "[\n"
+      for (resolvedUrl <- resolvedUrls) {
+        jsonString += toJsonString(resolvedUrl.drsResponse, resolvedUrl.downloadDestinationPath)
+      }
+      jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
+      jsonString += "\n]"
+      Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
     }
-    jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
-    jsonString += "\n]"
-    Try(Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8)))
   }
 
   def runGetm: IO[GetmResult] = {
-    val script = generateBulkDownloadScript
-    val copyCommand : Seq[String] = Seq("bash", "-c", script.get)
-    logger.info(script.get)
-    val copyProcess = Process(copyCommand)
-    val stderr = new StringBuilder()
-    val errorCapture: String => Unit = { s => stderr.append(s); () }
-    val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
-    logger.info(stderr.toString().trim())
-    IO(GetmResult(returnCode, stderr.toString().trim()))
+    generateJsonManifest(resolvedUrls).flatMap{ manifestPath =>
+      //val script = s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm --manifest '$manifestPath'""" //TODO: Check if getm will automatically create directories, or if we need to do it for each file.
+      // also consider deleting files already there to make retires a little simpler?
+      val manifestPathString = manifestPath.toString
+      val script = s"""getm --manifest '$manifestPathString'"""
+      val copyCommand : Seq[String] = Seq("bash", "-c", script)
+      logger.info(script)
+      val copyProcess = Process(copyCommand)
+      val stderr = new StringBuilder()
+      val errorCapture: String => Unit = { s => stderr.append(s); () }
+      val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
+      logger.info(stderr.toString().trim())
+      IO(GetmResult(returnCode, stderr.toString().trim()))
+    }
   }
 
   override def download: IO[DownloadResult] = {
@@ -91,14 +96,14 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
         DownloadSuccess
       case GetmResult(0, stderr) =>
         stderr match {
-          case ChecksumFailureMessage() =>
+          case BulkAccessUrlDownloader.ChecksumFailureMessage() =>
             ChecksumFailure
           case _ =>
             UnrecognizedRetryableDownloadFailure(ExitCode(0))
         }
       case GetmResult(rc, stderr) =>
         stderr match {
-          case HttpStatusMessage(status) =>
+          case BulkAccessUrlDownloader.HttpStatusMessage(status) =>
             Integer.parseInt(status) match {
               case 408 | 429 =>
                 RecognizedRetryableDownloadFailure(ExitCode(rc))
@@ -115,3 +120,10 @@ case class BulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String
     }
   }
 }
+
+object BulkAccessUrlDownloader{
+  type Hashes = Option[Map[String, String]]
+
+  val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r
+  val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r
+}
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
index 55e5ff4134a..82a239efc1a 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
@@ -1,13 +1,10 @@
 package drs.localizer.downloaders
 
 import cats.effect.IO
-import cloud.nio.impl.drs.AccessUrl
-import drs.localizer.downloaders.AccessUrlDownloader.Hashes
+import drs.localizer.ResolvedDrsUrlPendingDownload
 
 trait DownloaderFactory {
-  def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader]
-
-  def buildBulkAccessUrlDownloader(urlToDownloadDestinationMap : Map[AccessUrl, String]) : IO[Downloader]
+  def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrlPendingDownload]) : IO[Downloader]
 
   def buildGcsUriDownloader(gcsPath: String,
                             serviceAccountJsonOption: Option[String],
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
index 2a39a6543a3..2ca1bd3d2e3 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala
@@ -3,7 +3,7 @@ package drs.localizer.downloaders
 import cats.syntax.validated._
 import cloud.nio.impl.drs.AccessUrl
 import common.validation.ErrorOr.ErrorOr
-import drs.localizer.downloaders.AccessUrlDownloader.Hashes
+import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import org.apache.commons.codec.binary.Base64.encodeBase64String
 import org.apache.commons.codec.binary.Hex.decodeHex
 import org.apache.commons.text.StringEscapeUtils
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 0602a1dd73c..aee0884eeb4 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -8,7 +8,7 @@ import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsResolverFiel
 import common.assertion.CromwellTimeoutSpec
 import common.validation.ErrorOr.ErrorOr
 import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes}
-import drs.localizer.downloaders.AccessUrlDownloader.Hashes
+import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
@@ -91,10 +91,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     })
 
     val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        accessUrlDownloader
-      }
-
       override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala
index df7512dd81a..f820ed9cae2 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala
@@ -1,5 +1,5 @@
 package drs.localizer.downloaders
-
+/*
 import cats.effect.ExitCode
 import cats.syntax.validated._
 import cloud.nio.impl.drs.AccessUrl
@@ -7,7 +7,7 @@ import common.assertion.CromwellTimeoutSpec
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.TableDrivenPropertyChecks._
-
+//TODO: Migrate to Bulk Tests
 class AccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
   it should "return the correct download script for a url-only access URL, no requester pays" in {
     val fakeDownloadLocation = "/root/foo/foo-123.bam"
@@ -57,3 +57,4 @@ class AccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with
     }
   }
 }
+*/
\ No newline at end of file

From 6a77152ecb5bc53250c2ec26c76c2f1816f8ed6c Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 8 Sep 2023 15:26:49 -0400
Subject: [PATCH 17/36] time for tests

---
 .../drs/localizer/DrsLocalizerMain.scala      | 71 +++++++++----------
 .../downloaders/BulkAccessUrlDownloader.scala |  8 +--
 .../downloaders/DownloaderFactory.scala       |  4 +-
 .../BulkAccessUrlDownloaderSpec.scala         |  5 +-
 4 files changed, 43 insertions(+), 45 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index d8eaab4809b..2611984f7aa 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -17,9 +17,8 @@ import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import drs.localizer.URIType.URIType
 
-import scala.util.{Failure, Try}
 case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
-case class ResolvedDrsUrlPendingDownload(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
+case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
 
   override def run(args: List[String]): IO[ExitCode] = {
@@ -47,7 +46,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
       IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
 
-    override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrlPendingDownload]): IO[Downloader] = {
+    override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
       IO.pure(BulkAccessUrlDownloader(urlsToDownload))
     }
   }
@@ -83,10 +82,10 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       val exitCode : ExitCode = commandLineArguments.manifestPath match {
         case Some(manifestPath) => {
           val unresolvedUris: IO[List[UnresolvedDrsUrl]] = loadCSVManifest(manifestPath)
-          val resolvedUris: IO[List[ResolvedDrsUrlPendingDownload]] = unresolvedUris.flatMap(uris => uris.map(toResolve => resolveDrsUrl(resolver, toResolve)).traverse(identity))
+          val resolvedUris: IO[List[ResolvedDrsUrl]] = unresolvedUris.flatMap(uris => uris.map(toResolve => resolveDrsUrl(resolver, toResolve)).traverse(identity))
 
           // Resolve all the URLs, sorting them into buckets by their required downloader type
-          val resolvedUrisByType: Map[URIType, List[ResolvedDrsUrlPendingDownload]] = resolvedUris.map { resolvedList =>
+          val resolvedUrisByType: Map[URIType, List[ResolvedDrsUrl]] = resolvedUris.map { resolvedList =>
             Map(
               URIType.GCS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.GCS),
               URIType.HTTPS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.HTTPS),
@@ -94,26 +93,29 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
             )
           }.unsafeRunSync()
 
-          val googleDownload : IO[List[DownloadResult]] = resolvedUrisByType.get(URIType.GCS).map(googleUris => bulkDownloadResolvedGoogleUrls(googleUris, commandLineArguments.googleRequesterPaysProject)).getOrElse(List())
-          val bulkAccessDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.HTTPS).map(resolvedUris => bulkDownloadResolvedAccessUrls(resolvedUris)).getOrElse(IO(DownloadSuccess))
+          val googleDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.GCS).map(googleUris => bulkDownloadResolvedGoogleUrls(googleUris, commandLineArguments.googleRequesterPaysProject)).getOrElse(IO.pure(DownloadSuccess))
+          val bulkAccessDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.HTTPS).map(resolvedUris => bulkDownloadResolvedAccessUrls(resolvedUris)).getOrElse(IO.pure(DownloadSuccess))
           resolvedUrisByType.get(URIType.UNKNOWN).map(_ => logger.error("Unrecognized URI format returned from resolver"))
 
+          val googleResult = googleDownload.unsafeRunSync()
           val bulkAccessResult = bulkAccessDownload.unsafeRunSync()
-          val googleResults = googleDownload.unsafeRunSync()
-          yield if (googleResults.find(res => res != DownloadSuccess).isEmpty && bulkAccessResult.exitCode == ExitCode.Success) ExitCode.Success else ExitCode.Error
+          if(googleResult == DownloadSuccess && bulkAccessResult == DownloadSuccess) ExitCode.Success else ExitCode.Error
         }
-        case None =>
-          val drsObject = commandLineArguments.drsObject.get
-          val containerPath = commandLineArguments.containerPath.get
-          val resolvedURI = resolveDrsUrl(resolver, drsObject)
-          val map: Map[DrsResolverResponse, String] = Map((resolvedURI.unsafeRunSync() -> containerPath))
-          bulkDownloadResolvedAccessUrls(map, drsCredentials)
-        //TODO: Also handle the Google download case here.
+        case None => {
+          val maybeUrl : Option[UnresolvedDrsUrl] = commandLineArguments.drsObject.flatMap(obj => commandLineArguments.containerPath.map(path => UnresolvedDrsUrl(obj,path)))
+          maybeUrl.map(unresolved => {
+            val resolved: ResolvedDrsUrl = resolveDrsUrl(resolver, unresolved).unsafeRunSync()
+            val downloadResult: DownloadResult = if (resolved.uriType == URIType.HTTPS) bulkDownloadResolvedAccessUrls(List(resolved)).unsafeRunSync() else bulkDownloadResolvedGoogleUrls(List(resolved), commandLineArguments.googleRequesterPaysProject).unsafeRunSync()
+            if (downloadResult == DownloadSuccess) ExitCode.Success else ExitCode.Error
+          }).getOrElse(ExitCode.Error)
+          }
       }
+      exitCode
     }
+
   }
 
-  private def bulkDownloadResolvedAccessUrls(resolvedUrls : List[ResolvedDrsUrlPendingDownload]): IO[DownloadResult] = {
+  private def bulkDownloadResolvedAccessUrls(resolvedUrls : List[ResolvedDrsUrl]): IO[DownloadResult] = {
     IO[DownloadResult] {
       if(resolvedUrls.isEmpty) return IO(DownloadSuccess)
       val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls).unsafeRunSync()
@@ -121,34 +123,31 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
 
-  def bulkDownloadResolvedGoogleUrls(resolvedUrls : List[ResolvedDrsUrlPendingDownload], googleRequesterPaysProject : Option[String]) : IO[List[DownloadResult]] = {
-    resolvedUrls.map(url => {
-      val serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2)
-      val downloader = defaultDownloaderFactory.buildGcsUriDownloader(
-        gcsPath = url.drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
-        serviceAccountJsonOption = serviceAccountJsonOption,
-        downloadLoc = url.downloadDestinationPath,
-        requesterPaysProjectOption = googleRequesterPaysProject)
-      downloader.flatMap(_.download)
-    }).traverse(identity)
+  def downloadResolvedGoogleUrl(resolvedUrl : ResolvedDrsUrl, googleRequesterPaysProject : Option[String]) : IO[DownloadResult] = {
+    val serviceAccountJsonOption = resolvedUrl.drsResponse.googleServiceAccount.map(_.data.spaces2)
+    val downloader = defaultDownloaderFactory.buildGcsUriDownloader(
+      gcsPath = resolvedUrl.drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
+      serviceAccountJsonOption = serviceAccountJsonOption,
+      downloadLoc = resolvedUrl.downloadDestinationPath,
+      requesterPaysProjectOption = googleRequesterPaysProject)
+    downloader.flatMap(_.download)
+  }
+  def bulkDownloadResolvedGoogleUrls(resolvedUrls : List[ResolvedDrsUrl], googleRequesterPaysProject : Option[String]) : IO[DownloadResult] = {
+    val downloadIos : List[IO[DownloadResult]] = resolvedUrls.map(url => downloadResolvedGoogleUrl(url, googleRequesterPaysProject))
+    val downloadSuccesses : List[DownloadResult] = downloadIos.map(io => io.unsafeRunSync()).filter(res => res==DownloadSuccess)
+    val anyFailures = downloadIos.length != downloadSuccesses.length
+    if(anyFailures) IO.pure(FatalDownloadFailure(exitCode = ExitCode(1))) else IO.pure(DownloadSuccess)
   }
 
   //NB: This method calls a synchronous HTTP request
-  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrlPendingDownload] = {
+  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
     IO {
       val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
       //Insert retry logic here.
       val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
-      ResolvedDrsUrlPendingDownload(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
+      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
     }
   }
-  // resolve all the URIs once.
-  def resolveCSVManifest(csvManifestPath: String, resolver: DrsLocalizerDrsPathResolver) : Try[Map[DrsResolverResponse, String]] = {
-    val drsUriToDownloadLocationMap: Try[Map[String, String]] = loadCSVAsMap(csvManifestPath)
-    if (drsUriToDownloadLocationMap.isFailure) return Failure[Map[DrsResolverResponse, String]](new RuntimeException("Could not parse CSV manifest"))
-    val resolvedUriToDownloadLocationMap: Map[DrsResolverResponse, String] = drsUriToDownloadLocationMap.get.map(value => (resolveDrsUrl(resolver, value._1).unsafeRunSync(), value._2))
-    Try(resolvedUriToDownloadLocationMap)
-  }
 
   /**
     * Helper function to read a CSV file as a map from drs URL to requested download destination
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index e1502c53e0b..fbb3212009a 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -8,7 +8,7 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path, Paths}
 import scala.sys.process.{Process, ProcessLogger}
 import scala.util.matching.Regex
-import drs.localizer.ResolvedDrsUrlPendingDownload
+import drs.localizer.ResolvedDrsUrl
 case class GetmResult(returnCode: Int, stderr: String)
 
 /**
@@ -17,7 +17,7 @@ case class GetmResult(returnCode: Int, stderr: String)
   * to invoke the Getm tool, which is expected to already be installed in the local environment.
   * @param resolvedUrls
   */
-case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrlPendingDownload]) extends Downloader with StrictLogging {
+case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends Downloader with StrictLogging {
   /**
     * Write a json manifest to disk that looks like:
     * // [
@@ -38,11 +38,11 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrlPendingDown
     * @param resolvedUrls
     * @return Filepath of a getm-manifest.json that Getm can use to download multiple files in parallel.
     */
-  def generateJsonManifest(resolvedUrls : List[ResolvedDrsUrlPendingDownload]): IO[Path] = {
+  def generateJsonManifest(resolvedUrls : List[ResolvedDrsUrl]): IO[Path] = {
 
     def toJsonString(drsResponse: DrsResolverResponse, destinationFilepath: String): String = {
       //NB: trailing comma is being removed in generateJsonManifest
-      val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse("missing_url")
+      val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse(AccessUrl("missing", None))
       val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
       val checksumAlgorithm = checksum.getmAlgorithm
       s"""  {
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
index 82a239efc1a..c7caae74ec7 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
@@ -1,10 +1,10 @@
 package drs.localizer.downloaders
 
 import cats.effect.IO
-import drs.localizer.ResolvedDrsUrlPendingDownload
+import drs.localizer.ResolvedDrsUrl
 
 trait DownloaderFactory {
-  def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrlPendingDownload]) : IO[Downloader]
+  def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]) : IO[Downloader]
 
   def buildGcsUriDownloader(gcsPath: String,
                             serviceAccountJsonOption: Option[String],
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index 7f39f86ea15..be52bd32ab7 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -1,16 +1,14 @@
 package drs.localizer.downloaders
 
-import cloud.nio.impl.drs.AccessUrl
 import common.assertion.CromwellTimeoutSpec
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
-import java.nio.file.Path
-import scala.util.Try
 
 
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
 
+  /*
   it should "correctly parse a collection of Access Urls into a manifest.json" in {
 
     val fakeAccessURLToDownloadLocationMap : Map[AccessUrl, String] = Map(
@@ -42,4 +40,5 @@ class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec w
 
     lines shouldBe expected
   }
+   */
 }

From 00ed9dc17a3085227cb122cac18f77293063e010 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 8 Sep 2023 16:30:23 -0400
Subject: [PATCH 18/36] remove conf

---
 src/ci/resources/tes_application.conf | 75 +++++++++++++--------------
 1 file changed, 36 insertions(+), 39 deletions(-)

diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf
index 9a8642871ae..5eb812a52d4 100644
--- a/src/ci/resources/tes_application.conf
+++ b/src/ci/resources/tes_application.conf
@@ -2,36 +2,40 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
 filesystems {
-    blob {
-        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-        global {
-            class = "cromwell.filesystems.blob.BlobFileSystemManager"
-            config {
-                workspace-manager: {
-                    url: "https://workspace.dsde-dev.broadinstitute.org"
-                    workspace-id: "d4564046-bbba-495c-afec-14f7d3a8283a"
-                    container-resource-id: "9e7bdba5-047e-430c-810d-c7cb6645c155"
-                    b2cToken: "eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlCUHk1MzZwaHZmT2NlQUhjdzdOMG5OVmxSSDVwdkFyOWE2S3ZKbnFPQjQycVZkYzVOUDlJc2Jndk1iakNSZ2YycWRuaUhhMVpZOTlfdThMUTNFWGUzRG80SEhBQkctV2VEVTVmeHZOT1lRNjdNbS16cGRYN2JnOXl5WU5UcTJ3T2lObjNuMmh2bTBOdHBXR2VDOEJJekxZc0hLTzljU3RiS3NWUU5HVGNYa2FDZ1lLQWZZU0FSQVNGUUhzdllsczZoNTVrendUdlBMRTVPd2JMNktueVEwMTgzIiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTI3Mjc2NjEsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MjczMTI2MSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkyNzI3NjYxfQ.OyIZkNqiGQ_cIih0RYSBbin-BaG7B605MTH7RUIsZQOsLrkD1NR0U5q6tK0BO3A-NqwV4V26suJISGIHF4MlgdFji4p3499p-m7STZ8isQx3644n2jzpvjqaHgVxfYIofEMcDezOo7xZdCViq0UhiC3ajiWTgQmzy-q31L50yqz28ZiwAheTSNbcNyZtVE8ogImpYoem_3qfw-iJu3Sb0u_6OUgsvuiRoYbV-Sh2a4vIeyWlLI8Z6cW5Mrk-xzWSpp7EgnaaxYNye-0PLq5ziQV5DkxZ8XLupnn511wVJrYWkqJ91vEhuXfKLn8ZYnc7iUuJNxoJESRPj0PIit3Ysg"
-                }
-                container: "sc-d4564046-bbba-495c-afec-14f7d3a8283a"
-                endpoint: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net"
-            }
+  blob {
+    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+    global {
+      # One BFSM is shared across all BlobPathBuilders
+      class = "cromwell.filesystems.blob.BlobFileSystemManager"
+      config {
+        container: "cromwell"
+        endpoint: "https://<storage-account>.blob.core.windows.net"
+        subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
+        # WSM config is needed for accessing WSM-managed blob containers
+        # created in Terra workspaces.
+        workspace-manager {
+          url: "https://workspace.dsde-dev.broadinstitute.org"
+          workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+          container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
+          b2cToken: "Zardoz"
         }
+      }
     }
+  }
 }
 
 engine {
-    filesystems {
-        local {
-            enabled: true
-        }
-        http {
-            enabled: true
-        }
-        blob {
-            enabled: true
-        }
+  filesystems {
+    local {
+      enabled: true
+    }
+    http {
+      enabled: true
     }
+    blob {
+      enabled: false
+    }
+  }
 }
 
 
@@ -42,28 +46,26 @@ backend {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
         # Use for running on blob storage
-        root = "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-d4564046-bbba-495c-afec-14f7d3a8283a/workspace-services/cbas/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/"
+        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
+        root = "cromwell-executions"
         dockerRoot = "/cromwell-executions"
-        endpoint = "https://lzd98e904c9c1cee247af683bc764eb1b22f3912f8e7839afe.servicebus.windows.net/terra-app-566e92a0-e55e-4250-b4c1-d0925dd03916-d4564046-bbba-495c-afec-14f7d3a8283a/tes/v1/tasks"
+        endpoint = "http://127.0.0.1:9000/v1/tasks"
         concurrent-job-limit = 1000
-        workflow-execution-identity = "pet-2676571657071603a9eab"
-        bearer-token= "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Inp4UnJ6aEw4OTBrYi1vVkZOWkdWUFlxSzYtSkI3RFRlTGVvMllTbzRtc1UiLCJ0eXAiOiJKV1QifQ.eyJlbWFpbCI6InR3aXNlbWFuQGJyb2FkaW5zdGl0dXRlLm9yZyIsImdpdmVuX25hbWUiOiJUaG9tYXMiLCJmYW1pbHlfbmFtZSI6Ildpc2VtYW4iLCJuYW1lIjoiVGhvbWFzIFdpc2VtYW4iLCJpZHAiOiJnb29nbGUuY29tIiwiaWRwX2FjY2Vzc190b2tlbiI6InlhMjkuYTBBZkJfYnlCUHk1MzZwaHZmT2NlQUhjdzdOMG5OVmxSSDVwdkFyOWE2S3ZKbnFPQjQycVZkYzVOUDlJc2Jndk1iakNSZ2YycWRuaUhhMVpZOTlfdThMUTNFWGUzRG80SEhBQkctV2VEVTVmeHZOT1lRNjdNbS16cGRYN2JnOXl5WU5UcTJ3T2lObjNuMmh2bTBOdHBXR2VDOEJJekxZc0hLTzljU3RiS3NWUU5HVGNYa2FDZ1lLQWZZU0FSQVNGUUhzdllsczZoNTVrendUdlBMRTVPd2JMNktueVEwMTgzIiwiZ29vZ2xlX2lkIjoiMTExNDI1MTI3NzgyODk3OTE5MjIxIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FBY0hUdGNhZWxfZTl4dHdJQWdOZW1YY2N4d18xUWtlVjV1cF91NUFRTUMxMU5xNj1zOTYtYyIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJzdWIiOiJhMzYxY2YxNS0xNTgxLTQ2ZDQtYmJkNC1mMGQ0NGFlNjkwODciLCJ0aWQiOiJmZDBiYzBlZi0xNzQ3LTRlZTYtYWIzZS1kNGQ2YmI4ODJkNDAiLCJhenAiOiJiYmQwN2Q0My0wMWNiLTRiNjktOGZkMC01NzQ2ZDlhNWM5ZmUiLCJ2ZXIiOiIxLjAiLCJpYXQiOjE2OTI3Mjc2NjEsImF1ZCI6ImJiZDA3ZDQzLTAxY2ItNGI2OS04ZmQwLTU3NDZkOWE1YzlmZSIsImV4cCI6MTY5MjczMTI2MSwiaXNzIjoiaHR0cHM6Ly90ZXJyYWRldmIyYy5iMmNsb2dpbi5jb20vZmQwYmMwZWYtMTc0Ny00ZWU2LWFiM2UtZDRkNmJiODgyZDQwL3YyLjAvIiwibmJmIjoxNjkyNzI3NjYxfQ.OyIZkNqiGQ_cIih0RYSBbin-BaG7B605MTH7RUIsZQOsLrkD1NR0U5q6tK0BO3A-NqwV4V26suJISGIHF4MlgdFji4p3499p-m7STZ8isQx3644n2jzpvjqaHgVxfYIofEMcDezOo7xZdCViq0UhiC3ajiWTgQmzy-q31L50yqz28ZiwAheTSNbcNyZtVE8ogImpYoem_3qfw-iJu3Sb0u_6OUgsvuiRoYbV-Sh2a4vIeyWlLI8Z6cW5Mrk-xzWSpp7EgnaaxYNye-0PLq5ziQV5DkxZ8XLupnn511wVJrYWkqJ91vEhuXfKLn8ZYnc7iUuJNxoJESRPj0PIit3Ysg"
+
         # Override default polling to make it faster for speedy tests
         poll-backoff {
-          min: "10 seconds"
+          min: "1 seconds"
           max: "5 minutes"
           multiplier: 1.1
           randomization-factor: 0.5
         }
-
-        use_tes_11_preview_backend_parameters = true
-
+        
         filesystems {
           blob {
-            enabled: true
+            enabled: false
           }
           local {
-            enabled: false
+            enabled: true
           }
           http {
             enabled: true
@@ -76,8 +78,3 @@ backend {
     }
   }
 }
-
-workflow-options {
-  # Directory where to write per workflow logs
-  workflow-log-dir: "https://lz43a8a3d21540dfd25f5ace.blob.core.windows.net/sc-ebda3e1d-09bf-48b1-be2e-3d09f67dfc5c/workspace-services/cbas/terra-app-4628d0e1-d4b5-4b09-ad5c-9a5766650a7a/workflow-logs"
-}
\ No newline at end of file

From 510b085e491db216790c253b2c890f8aafaf7917 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 8 Sep 2023 17:04:21 -0400
Subject: [PATCH 19/36] working on tests

---
 .../downloaders/BulkAccessUrlDownloader.scala | 28 +++++++++++--------
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 13 +++++++--
 .../BulkAccessUrlDownloaderSpec.scala         | 28 +++++++++++--------
 3 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index fbb3212009a..036e6d64253 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -39,21 +39,27 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
     * @return Filepath of a getm-manifest.json that Getm can use to download multiple files in parallel.
     */
   def generateJsonManifest(resolvedUrls : List[ResolvedDrsUrl]): IO[Path] = {
-
     def toJsonString(drsResponse: DrsResolverResponse, destinationFilepath: String): String = {
       //NB: trailing comma is being removed in generateJsonManifest
       val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse(AccessUrl("missing", None))
-      val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
-      val checksumAlgorithm = checksum.getmAlgorithm
-      s"""  {
-         |    "url" : "$accessUrl",
-         |    "filepath" : "$destinationFilepath",
-         |    "checksum" : "$checksum",
-         |    "checksum-algorithm" : "$checksumAlgorithm"
-         |  },
-         |""".stripMargin
+      drsResponse.hashes.map(_ => {
+        val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
+        val checksumAlgorithm = checksum.getmAlgorithm
+        s"""  {
+           |    "url" : "$accessUrl",
+           |    "filepath" : "$destinationFilepath",
+           |    "checksum" : "$checksum",
+           |    "checksum-algorithm" : "$checksumAlgorithm"
+           |  },
+           |""".stripMargin
+      }).getOrElse(
+        s"""  {
+           |    "url" : "$accessUrl",
+           |    "filepath" : "$destinationFilepath"
+           |  },
+           |""".stripMargin
+      )
     }
-
     IO {
       var jsonString: String = "[\n"
       for (resolvedUrl <- resolvedUrls) {
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index aee0884eeb4..2538899333b 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -1,5 +1,6 @@
 package drs.localizer
 
+/*
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO}
 import cats.syntax.validated._
@@ -12,8 +13,8 @@ import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-
-
+*/
+/*
 class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
 
   val fakeDownloadLocation = "/root/foo/foo-123.bam"
@@ -315,6 +316,9 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
   }
 }
+ */
+
+/*
 
 object MockDrsPaths {
   val fakeDrsUrlWithGcsResolutionOnly = "drs://abc/foo-123/abc123"
@@ -328,6 +332,7 @@ class MockDrsLocalizerMain(drsUrl: String,
                            downloadLoc: String,
                            requesterPaysProjectIdOption: Option[String],
                           )
+
   extends DrsLocalizerMain(drsUrl, downloadLoc, FakeAccessTokenStrategy, requesterPaysProjectIdOption) {
 
   override def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
@@ -370,4 +375,8 @@ object MockDrsLocalizerDrsPathResolver {
   val FakeAccessTokenStrategy: DrsCredentials = new DrsCredentials {
     override def getAccessToken: ErrorOr[String] = "testing code: do not call me".invalidNel
   }
+
 }
+
+
+ */
\ No newline at end of file
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index be52bd32ab7..fa1a68a90d1 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -1,21 +1,27 @@
 package drs.localizer.downloaders
 
+import cats.effect.IO
+import cloud.nio.impl.drs.DrsResolverField.AccessUrl
+import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
+import drs.localizer.{ResolvedDrsUrl, URIType}
+//import drs.localizer.URIType.URIType
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
+import java.nio.file.Path
+
 
 
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
+  val emptyList : List[ResolvedDrsUrl] = List()
+  val oneElement: List[ResolvedDrsUrl] = List(ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest", URIType.HTTPS))
+  val threeElements: List[ResolvedDrsUrl] = List(
+    ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.HTTPS),
+    ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest2", URIType.HTTPS),
+    ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest3", URIType.HTTPS))
 
-  /*
   it should "correctly parse a collection of Access Urls into a manifest.json" in {
-
-    val fakeAccessURLToDownloadLocationMap : Map[AccessUrl, String] = Map(
-      (AccessUrl("https://my.fake/url123", None) -> "./test1.out"),
-      (AccessUrl("https://my.fake/url1234", None) -> "./test2.out")
-    )
-
     val expected : String =
       s"""[
          |  {
@@ -32,13 +38,13 @@ class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec w
          |  }
          |]""".stripMargin
 
-    val downloader = BulkAccessUrlDownloader(fakeAccessURLToDownloadLocationMap)
+    val downloader = BulkAccessUrlDownloader(threeElements)
 
-    val filepath : Try[Path] = downloader.generateJsonManifest(fakeAccessURLToDownloadLocationMap)
-    val source = scala.io.Source.fromFile(filepath.get.toString)
+    val filepath : IO[Path] = downloader.generateJsonManifest(threeElements)
+    val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString)
     val lines = try source.mkString finally source.close()
 
     lines shouldBe expected
   }
-   */
+
 }

From 8b99bac59d622729abfcb3f8e114cde26e1f77f6 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Mon, 11 Sep 2023 14:04:27 -0400
Subject: [PATCH 20/36] test progress

---
 .../downloaders/BulkAccessUrlDownloader.scala | 16 ++--
 .../BulkAccessUrlDownloaderSpec.scala         | 78 +++++++++++++------
 2 files changed, 67 insertions(+), 27 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index 036e6d64253..b5b50909c7d 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -46,7 +46,7 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
         val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
         val checksumAlgorithm = checksum.getmAlgorithm
         s"""  {
-           |    "url" : "$accessUrl",
+           |    "url" : "${accessUrl.url}",
            |    "filepath" : "$destinationFilepath",
            |    "checksum" : "$checksum",
            |    "checksum-algorithm" : "$checksumAlgorithm"
@@ -54,7 +54,7 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
            |""".stripMargin
       }).getOrElse(
         s"""  {
-           |    "url" : "$accessUrl",
+           |    "url" : "${accessUrl.url}",
            |    "filepath" : "$destinationFilepath"
            |  },
            |""".stripMargin
@@ -65,18 +65,24 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
       for (resolvedUrl <- resolvedUrls) {
         jsonString += toJsonString(resolvedUrl.drsResponse, resolvedUrl.downloadDestinationPath)
       }
-      jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) //remove trailing comma from array elements
+      if(jsonString.contains(',')) {
+        //remove trailing comma from array elements, but don't crash on empty list.
+        jsonString = jsonString.substring(0, jsonString.lastIndexOf(","))
+      }
       jsonString += "\n]"
       Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
     }
   }
 
+  def generateGetmCommand(pathToMainfestJson : Path) : String = {
+    s"""getm --manifest ${pathToMainfestJson.toString}"""
+  }
   def runGetm: IO[GetmResult] = {
     generateJsonManifest(resolvedUrls).flatMap{ manifestPath =>
       //val script = s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm --manifest '$manifestPath'""" //TODO: Check if getm will automatically create directories, or if we need to do it for each file.
       // also consider deleting files already there to make retires a little simpler?
-      val manifestPathString = manifestPath.toString
-      val script = s"""getm --manifest '$manifestPathString'"""
+
+      val script = generateGetmCommand(manifestPath)
       val copyCommand : Seq[String] = Seq("bash", "-c", script)
       logger.info(script)
       val copyProcess = Process(copyCommand)
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index fa1a68a90d1..ddaa301fde8 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -1,7 +1,6 @@
 package drs.localizer.downloaders
 
 import cats.effect.IO
-import cloud.nio.impl.drs.DrsResolverField.AccessUrl
 import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
 import drs.localizer.{ResolvedDrsUrl, URIType}
@@ -9,42 +8,77 @@ import drs.localizer.{ResolvedDrsUrl, URIType}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
+
 import java.nio.file.Path
 
 
 
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
+  val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.HTTPS)
+  val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), "path/to/local/download/dest2", URIType.HTTPS)
+  val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), "path/to/local/download/dest3", URIType.HTTPS)
   val emptyList : List[ResolvedDrsUrl] = List()
-  val oneElement: List[ResolvedDrsUrl] = List(ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest", URIType.HTTPS))
-  val threeElements: List[ResolvedDrsUrl] = List(
-    ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.HTTPS),
-    ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest2", URIType.HTTPS),
-    ResolvedDrsUrl(DrsResolverResponse(), "path/to/local/download/dest3", URIType.HTTPS))
+  val oneElement: List[ResolvedDrsUrl] = List(ex1)
+  val threeElements: List[ResolvedDrsUrl] = List(ex1, ex2, ex3)
 
   it should "correctly parse a collection of Access Urls into a manifest.json" in {
-    val expected : String =
-      s"""[
-         |  {
-         |    "url" : "https://my.fake/url123",
-         |    "filepath" : "./test1.out",
-         |    "checksum" : "Valid()",
-         |    "checksum-algorithm" : "null"
-         |  },
-         |  {
-         |    "url" : "https://my.fake/url1234",
-         |    "filepath" : "./test2.out",
-         |    "checksum" : "Valid()",
-         |    "checksum-algorithm" : "null"
-         |  }
-         |]""".stripMargin
+    val expected: String =
+      s"""|[
+          |  {
+          |    "url" : "https://my.fake/url123",
+          |    "filepath" : "path/to/local/download/dest"
+          |  },
+          |  {
+          |    "url" : "https://my.fake/url1234",
+          |    "filepath" : "path/to/local/download/dest2"
+          |  },
+          |  {
+          |    "url" : "https://my.fake/url1235",
+          |    "filepath" : "path/to/local/download/dest3"
+          |  }
+          |]""".stripMargin
 
     val downloader = BulkAccessUrlDownloader(threeElements)
 
-    val filepath : IO[Path] = downloader.generateJsonManifest(threeElements)
+    val filepath: IO[Path] = downloader.generateJsonManifest(threeElements)
     val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString)
     val lines = try source.mkString finally source.close()
+    lines shouldBe expected
+  }
+
+  it should "properly construct empty JSON array from empty list." in {
+    val expected: String =
+      s"""|[
+          |
+          |]""".stripMargin
 
+    val downloader = BulkAccessUrlDownloader(emptyList)
+    val filepath: IO[Path] = downloader.generateJsonManifest(emptyList)
+    val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString)
+    val lines = try source.mkString finally source.close()
     lines shouldBe expected
   }
 
+  it should "properly construct JSON array from single element list." in {
+    val expected: String =
+      s"""|[
+          |  {
+          |    "url" : "https://my.fake/url123",
+          |    "filepath" : "path/to/local/download/dest"
+          |  }
+          |]""".stripMargin
+
+    val downloader = BulkAccessUrlDownloader(oneElement)
+    val filepath: IO[Path] = downloader.generateJsonManifest(oneElement)
+    val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString)
+    val lines = try source.mkString finally source.close()
+    lines shouldBe expected
+  }
+
+  it should "properly construct the invocation command" in {
+    val downloader = BulkAccessUrlDownloader(oneElement)
+    val filepath: Path = downloader.generateJsonManifest(threeElements).unsafeRunSync()
+    val expected = s"""getm --manifest ${filepath.toString}"""
+    downloader.generateGetmCommand(filepath) shouldBe expected
+  }
 }

From 99efe34e9c23ecbf9db40d13390157fb18bfaf89 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Mon, 11 Sep 2023 16:43:53 -0400
Subject: [PATCH 21/36] much cleaner

---
 .../drs/localizer/DrsLocalizerMain.scala      | 176 +++++++++---------
 .../drs/localizer/DrsLocalizerMainSpec.scala  |  11 +-
 .../BulkAccessUrlDownloaderSpec.scala         |   5 -
 3 files changed, 85 insertions(+), 107 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 2611984f7aa..1abe6345cfc 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -16,7 +16,6 @@ import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import drs.localizer.URIType.URIType
-
 case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
 case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
@@ -56,7 +55,44 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     IO.pure(ExitCode.Error)
   }
 
-  private def getDrsPathResolver(drsCredentials: DrsCredentials): IO[DrsLocalizerDrsPathResolver] = {
+  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials) : IO[ExitCode] = {
+    val urlList : IO[List[UnresolvedDrsUrl]] = commandLineArguments.manifestPath match {
+      case Some(manifestPath) =>
+        loadCSVManifest(manifestPath)
+      case None =>
+        IO.pure(List(UnresolvedDrsUrl(commandLineArguments.drsObject.get, commandLineArguments.containerPath.get)))
+      }
+    IO{
+      val main = new DrsLocalizerMain(urlList, drsCredentials, commandLineArguments.googleRequesterPaysProject)
+      main.resolveAndDownload(defaultDownloaderFactory).unsafeRunSync().exitCode
+    }
+    }
+  }
+
+/**
+  * Helper function to read a CSV file as a map from drs URL to requested download destination
+  *
+  * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
+  */
+def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = {
+  IO {
+    val openFile = new File(csvManifestPath)
+    val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
+    val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
+    list
+  }
+
+}
+
+object URIType extends Enumeration {
+  type URIType = Value
+  val GCS, HTTPS, UNKNOWN = Value
+}
+
+class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
+                       drsCredentials: DrsCredentials,
+                       requesterPaysProjectIdOption: Option[String]) extends StrictLogging {
+  def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
     IO {
       val drsConfig = DrsConfig.fromEnv(sys.env)
       logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects")
@@ -64,114 +100,68 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
 
-  private[localizer] def toUriType(accessUrl: Option[AccessUrl], gsUri : Option[String]): URIType = {
+  def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]) : IO[List[ResolvedDrsUrl]] = {
+    unresolvedUrls.flatMap{unresolvedList =>
+      getDrsPathResolver.flatMap{resolver =>
+        unresolvedList.map{unresolvedUrl =>
+          resolveSingleUrl(resolver, unresolvedUrl)
+        }.traverse(identity)
+      }
+    }
+  }
+
+  def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
+    IO {
+      val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+      //Insert retry logic here.
+      val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
+      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
+    }
+  }
+
+  private[localizer] def toUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = {
     (accessUrl, gsUri) match {
       case (_, Some(_)) =>
         URIType.GCS
       case (Some(_), _) =>
         URIType.HTTPS
-      case (_,_) =>
+      case (_, _) =>
         URIType.UNKNOWN
     }
   }
-  def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = {
-    IO {
-      // Get a single resolver to resolve URLs with
-      val resolver = getDrsPathResolver(drsCredentials).unsafeRunSync()
-
-      val exitCode : ExitCode = commandLineArguments.manifestPath match {
-        case Some(manifestPath) => {
-          val unresolvedUris: IO[List[UnresolvedDrsUrl]] = loadCSVManifest(manifestPath)
-          val resolvedUris: IO[List[ResolvedDrsUrl]] = unresolvedUris.flatMap(uris => uris.map(toResolve => resolveDrsUrl(resolver, toResolve)).traverse(identity))
-
-          // Resolve all the URLs, sorting them into buckets by their required downloader type
-          val resolvedUrisByType: Map[URIType, List[ResolvedDrsUrl]] = resolvedUris.map { resolvedList =>
-            Map(
-              URIType.GCS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.GCS),
-              URIType.HTTPS -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.HTTPS),
-              URIType.UNKNOWN -> resolvedList.filter(resolvedUri => resolvedUri.uriType == URIType.UNKNOWN)
-            )
-          }.unsafeRunSync()
-
-          val googleDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.GCS).map(googleUris => bulkDownloadResolvedGoogleUrls(googleUris, commandLineArguments.googleRequesterPaysProject)).getOrElse(IO.pure(DownloadSuccess))
-          val bulkAccessDownload : IO[DownloadResult] = resolvedUrisByType.get(URIType.HTTPS).map(resolvedUris => bulkDownloadResolvedAccessUrls(resolvedUris)).getOrElse(IO.pure(DownloadSuccess))
-          resolvedUrisByType.get(URIType.UNKNOWN).map(_ => logger.error("Unrecognized URI format returned from resolver"))
-
-          val googleResult = googleDownload.unsafeRunSync()
-          val bulkAccessResult = bulkAccessDownload.unsafeRunSync()
-          if(googleResult == DownloadSuccess && bulkAccessResult == DownloadSuccess) ExitCode.Success else ExitCode.Error
+
+  def resolveAndDownload(downloaderFactory: DownloaderFactory) : IO[DownloadResult] = {
+      resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
+        val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.HTTPS)
+        val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
+        pendingDownloads.filter(url => url.uriType == URIType.UNKNOWN).map(
+          unknown => logger.error(
+            s"Url is not an https:// or  gs:// URL: " +
+              s"${unknown.drsResponse.accessUrl.getOrElse(unknown.drsResponse.gsUri.getOrElse("missing_url"))}"))
+        IO{
+          val accessResults = bulkDownloadAccessUrls(accessUrls, downloaderFactory)
+          val googleResults = bulkDownloadGoogleUrls(accessUrls, downloaderFactory)
+          List(accessResults, googleResults).find(result => result != DownloadSuccess).getOrElse(DownloadSuccess)
         }
-        case None => {
-          val maybeUrl : Option[UnresolvedDrsUrl] = commandLineArguments.drsObject.flatMap(obj => commandLineArguments.containerPath.map(path => UnresolvedDrsUrl(obj,path)))
-          maybeUrl.map(unresolved => {
-            val resolved: ResolvedDrsUrl = resolveDrsUrl(resolver, unresolved).unsafeRunSync()
-            val downloadResult: DownloadResult = if (resolved.uriType == URIType.HTTPS) bulkDownloadResolvedAccessUrls(List(resolved)).unsafeRunSync() else bulkDownloadResolvedGoogleUrls(List(resolved), commandLineArguments.googleRequesterPaysProject).unsafeRunSync()
-            if (downloadResult == DownloadSuccess) ExitCode.Success else ExitCode.Error
-          }).getOrElse(ExitCode.Error)
-          }
       }
-      exitCode
-    }
-
   }
 
-  private def bulkDownloadResolvedAccessUrls(resolvedUrls : List[ResolvedDrsUrl]): IO[DownloadResult] = {
-    IO[DownloadResult] {
-      if(resolvedUrls.isEmpty) return IO(DownloadSuccess)
-      val downloader = defaultDownloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls).unsafeRunSync()
-      downloader.download.unsafeRunSync()
+  def bulkDownloadGoogleUrls(resolvedUrls: List[ResolvedDrsUrl], downloaderFactory: DownloaderFactory) = {
+    val downloadResults = resolvedUrls.map{url =>
+      downloaderFactory.buildGcsUriDownloader(
+        gcsPath = url.drsResponse.gsUri.get,
+        serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2),
+        downloadLoc = url.downloadDestinationPath,
+        requesterPaysProjectOption = requesterPaysProjectIdOption).flatMap(_.download).unsafeRunSync()
     }
+    downloadResults.find(res => res != DownloadSuccess).getOrElse(DownloadSuccess)
   }
 
-  def downloadResolvedGoogleUrl(resolvedUrl : ResolvedDrsUrl, googleRequesterPaysProject : Option[String]) : IO[DownloadResult] = {
-    val serviceAccountJsonOption = resolvedUrl.drsResponse.googleServiceAccount.map(_.data.spaces2)
-    val downloader = defaultDownloaderFactory.buildGcsUriDownloader(
-      gcsPath = resolvedUrl.drsResponse.gsUri.getOrElse("MISSING_GOOGLE_URI"),
-      serviceAccountJsonOption = serviceAccountJsonOption,
-      downloadLoc = resolvedUrl.downloadDestinationPath,
-      requesterPaysProjectOption = googleRequesterPaysProject)
-    downloader.flatMap(_.download)
-  }
-  def bulkDownloadResolvedGoogleUrls(resolvedUrls : List[ResolvedDrsUrl], googleRequesterPaysProject : Option[String]) : IO[DownloadResult] = {
-    val downloadIos : List[IO[DownloadResult]] = resolvedUrls.map(url => downloadResolvedGoogleUrl(url, googleRequesterPaysProject))
-    val downloadSuccesses : List[DownloadResult] = downloadIos.map(io => io.unsafeRunSync()).filter(res => res==DownloadSuccess)
-    val anyFailures = downloadIos.length != downloadSuccesses.length
-    if(anyFailures) IO.pure(FatalDownloadFailure(exitCode = ExitCode(1))) else IO.pure(DownloadSuccess)
+  def bulkDownloadAccessUrls(resolvedUrls: List[ResolvedDrsUrl], downloaderFactory: DownloaderFactory) : DownloadResult = {
+    if (resolvedUrls.isEmpty) return DownloadSuccess
+    downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls).unsafeRunSync().download.unsafeRunSync()
   }
 
-  //NB: This method calls a synchronous HTTP request
-  def resolveDrsUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
-    IO {
-      val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-      //Insert retry logic here.
-      val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
-      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
-    }
-  }
-
-  /**
-    * Helper function to read a CSV file as a map from drs URL to requested download destination
-    * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
-    */
-  private def loadCSVManifest(csvManifestPath : String) : IO[List[UnresolvedDrsUrl]] = {
-    IO {
-      val openFile = new File(csvManifestPath)
-      val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
-      val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
-      list
-    }
-  }
-}
-
-object URIType extends Enumeration {
-  type URIType = Value
-  val GCS, HTTPS, UNKNOWN = Value
-}
-
-class DrsLocalizerMain(drsUrl: String,
-                       downloadLoc: String,
-                       drsCredentials: DrsCredentials,
-                       requesterPaysProjectIdOption: Option[String]) extends StrictLogging {
   /*
   def resolveAndDownloadWithRetries(downloadRetries: Int,
                                     checksumRetries: Int,
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 2538899333b..11d4d64e1ec 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -1,6 +1,5 @@
 package drs.localizer
 
-/*
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO}
 import cats.syntax.validated._
@@ -13,8 +12,8 @@ import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-*/
-/*
+
+
 class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
 
   val fakeDownloadLocation = "/root/foo/foo-123.bam"
@@ -316,9 +315,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
   }
 }
- */
-
-/*
 
 object MockDrsPaths {
   val fakeDrsUrlWithGcsResolutionOnly = "drs://abc/foo-123/abc123"
@@ -377,6 +373,3 @@ object MockDrsLocalizerDrsPathResolver {
   }
 
 }
-
-
- */
\ No newline at end of file
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index ddaa301fde8..20b5e4c8aa0 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -4,15 +4,10 @@ import cats.effect.IO
 import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
 import drs.localizer.{ResolvedDrsUrl, URIType}
-//import drs.localizer.URIType.URIType
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-
-
 import java.nio.file.Path
 
-
-
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
   val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.HTTPS)
   val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), "path/to/local/download/dest2", URIType.HTTPS)

From 776bface9f2cc6c7f99beca38617d4c55d51f973 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Mon, 11 Sep 2023 16:49:10 -0400
Subject: [PATCH 22/36] oops

---
 .../src/main/scala/drs/localizer/DrsLocalizerMain.scala         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 1abe6345cfc..dddd3915c20 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -140,7 +140,7 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
               s"${unknown.drsResponse.accessUrl.getOrElse(unknown.drsResponse.gsUri.getOrElse("missing_url"))}"))
         IO{
           val accessResults = bulkDownloadAccessUrls(accessUrls, downloaderFactory)
-          val googleResults = bulkDownloadGoogleUrls(accessUrls, downloaderFactory)
+          val googleResults = bulkDownloadGoogleUrls(googleUrls, downloaderFactory)
           List(accessResults, googleResults).find(result => result != DownloadSuccess).getOrElse(DownloadSuccess)
         }
       }

From a05f63e8ac3dc9b979110269d23c17fb1aa14551 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 12 Sep 2023 14:01:26 -0400
Subject: [PATCH 23/36] bunch of tests

---
 .../drs/localizer/DrsLocalizerMain.scala      | 102 +--
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 586 +++++++++++-------
 2 files changed, 433 insertions(+), 255 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index dddd3915c20..9b36152ac14 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -7,6 +7,7 @@ import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
+import drs.localizer.DrsLocalizerMain.toUriType
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
 
@@ -55,6 +56,21 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     IO.pure(ExitCode.Error)
   }
 
+  /**
+    * Helper function to read a CSV file as a map from drs URL to requested download destination
+    *
+    * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
+    */
+  def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = {
+    IO {
+      val openFile = new File(csvManifestPath)
+      val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
+      val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
+      list
+    }
+  }
+
+
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials) : IO[ExitCode] = {
     val urlList : IO[List[UnresolvedDrsUrl]] = commandLineArguments.manifestPath match {
       case Some(manifestPath) =>
@@ -63,26 +79,22 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
         IO.pure(List(UnresolvedDrsUrl(commandLineArguments.drsObject.get, commandLineArguments.containerPath.get)))
       }
     IO{
-      val main = new DrsLocalizerMain(urlList, drsCredentials, commandLineArguments.googleRequesterPaysProject)
-      main.resolveAndDownload(defaultDownloaderFactory).unsafeRunSync().exitCode
+      val main = new DrsLocalizerMain(urlList, defaultDownloaderFactory, drsCredentials, commandLineArguments.googleRequesterPaysProject)
+      main.resolveAndDownload().unsafeRunSync().exitCode
     }
     }
-  }
 
-/**
-  * Helper function to read a CSV file as a map from drs URL to requested download destination
-  *
-  * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
-  */
-def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = {
-  IO {
-    val openFile = new File(csvManifestPath)
-    val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
-    val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
-    list
+  def toUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = {
+    (accessUrl, gsUri) match {
+      case (_, Some(_)) =>
+        URIType.GCS
+      case (Some(_), _) =>
+        URIType.HTTPS
+      case (_, _) =>
+        URIType.UNKNOWN
+    }
+  }
   }
-
-}
 
 object URIType extends Enumeration {
   type URIType = Value
@@ -90,6 +102,7 @@ object URIType extends Enumeration {
 }
 
 class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
+                       downloaderFactory: DownloaderFactory,
                        drsCredentials: DrsCredentials,
                        requesterPaysProjectIdOption: Option[String]) extends StrictLogging {
   def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
@@ -119,49 +132,44 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     }
   }
 
-  private[localizer] def toUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = {
-    (accessUrl, gsUri) match {
-      case (_, Some(_)) =>
-        URIType.GCS
-      case (Some(_), _) =>
-        URIType.HTTPS
-      case (_, _) =>
-        URIType.UNKNOWN
+
+
+  def resolveAndDownload() : IO[DownloadResult] = {
+    IO {
+      val downloaders: List[Downloader] = buildDownloaders().unsafeRunSync()
+      val results: List[DownloadResult] = downloaders.map(downloader => downloader.download.unsafeRunSync())
+      results.find(res => res!= DownloadSuccess).getOrElse(DownloadSuccess)
     }
   }
-
-  def resolveAndDownload(downloaderFactory: DownloaderFactory) : IO[DownloadResult] = {
-      resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
-        val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.HTTPS)
-        val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
-        pendingDownloads.filter(url => url.uriType == URIType.UNKNOWN).map(
-          unknown => logger.error(
-            s"Url is not an https:// or  gs:// URL: " +
-              s"${unknown.drsResponse.accessUrl.getOrElse(unknown.drsResponse.gsUri.getOrElse("missing_url"))}"))
-        IO{
-          val accessResults = bulkDownloadAccessUrls(accessUrls, downloaderFactory)
-          val googleResults = bulkDownloadGoogleUrls(googleUrls, downloaderFactory)
-          List(accessResults, googleResults).find(result => result != DownloadSuccess).getOrElse(DownloadSuccess)
-        }
-      }
+  def buildDownloaders() : IO[List[Downloader]] = {
+    resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
+      val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.HTTPS)
+      val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
+      pendingDownloads.filter(url => url.uriType == URIType.UNKNOWN).map(
+        unknown => logger.error(
+          s"Url is not an https:// or  gs:// URL: " +
+            s"${unknown.drsResponse.accessUrl.getOrElse(unknown.drsResponse.gsUri.getOrElse("missing_url"))}"))
+      val bulkDownloader: Option[List[IO[Downloader]]] = if(accessUrls.isEmpty) None else Option(List(buildBulkAccessUrlDownloader(accessUrls)))
+      val googleDownloaders: Option[List[IO[Downloader]]] = if(googleUrls.isEmpty) None else  Option(buildGoogleDownloaders(googleUrls))
+      val combined: List[IO[Downloader]] = googleDownloaders.map(list => list).getOrElse(List()) ++ bulkDownloader.map(list => list).getOrElse(List())
+      combined.traverse(identity)
+    }
   }
 
-  def bulkDownloadGoogleUrls(resolvedUrls: List[ResolvedDrsUrl], downloaderFactory: DownloaderFactory) = {
-    val downloadResults = resolvedUrls.map{url =>
+  def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]) : List[IO[Downloader]] = {
+    resolvedGoogleUrls.map{url=>
       downloaderFactory.buildGcsUriDownloader(
         gcsPath = url.drsResponse.gsUri.get,
         serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2),
         downloadLoc = url.downloadDestinationPath,
-        requesterPaysProjectOption = requesterPaysProjectIdOption).flatMap(_.download).unsafeRunSync()
+        requesterPaysProjectOption = requesterPaysProjectIdOption)
     }
-    downloadResults.find(res => res != DownloadSuccess).getOrElse(DownloadSuccess)
   }
-
-  def bulkDownloadAccessUrls(resolvedUrls: List[ResolvedDrsUrl], downloaderFactory: DownloaderFactory) : DownloadResult = {
-    if (resolvedUrls.isEmpty) return DownloadSuccess
-    downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls).unsafeRunSync().download.unsafeRunSync()
+  def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) : IO[Downloader] = {
+    downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls)
   }
 
+
   /*
   def resolveAndDownloadWithRetries(downloadRetries: Int,
                                     checksumRetries: Int,
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 11d4d64e1ec..4ded6ec2ff7 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -3,12 +3,13 @@ package drs.localizer
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO}
 import cats.syntax.validated._
-import cloud.nio.impl.drs.DrsPathResolver.FatalRetryDisposition
+import drs.localizer.MockDrsPaths.{fakeAccessUrls, fakeDrsUrlWithGcsResolutionOnly, fakeGoogleUrls}
+//import cloud.nio.impl.drs.DrsPathResolver.FatalRetryDisposition
 import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsResolverField, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
 import common.validation.ErrorOr.ErrorOr
 import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes}
-import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
+//import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
@@ -19,6 +20,28 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
   val fakeDownloadLocation = "/root/foo/foo-123.bam"
   val fakeRequesterPaysId = "fake-billing-project"
 
+  val fakeGoogleInput : IO[List[UnresolvedDrsUrl]] = IO(List(
+    UnresolvedDrsUrl(fakeDrsUrlWithGcsResolutionOnly, "/path/to/nowhere")
+  ))
+
+  val fakeAccessInput: IO[List[UnresolvedDrsUrl]] = IO(List(
+    UnresolvedDrsUrl("https://my-fake-access-url.com", "/path/to/somewhereelse")
+  ))
+
+  val fakeBulkGoogleInput: IO[List[UnresolvedDrsUrl]] = IO(List(
+    UnresolvedDrsUrl("drs://my-fake-google-url.com", "/path/to/nowhere"),
+    UnresolvedDrsUrl("drs://my-fake-google-url.com2", "/path/to/nowhere2"),
+    UnresolvedDrsUrl("drs://my-fake-google-url.com3", "/path/to/nowhere3"),
+    UnresolvedDrsUrl("drs://my-fake-google-url.com4", "/path/to/nowhere4")
+  ))
+
+  val fakeBulkAccessInput: IO[List[UnresolvedDrsUrl]] = IO(List(
+    UnresolvedDrsUrl("drs://my-fake-access-url.com", "/path/to/somewhereelse"),
+    UnresolvedDrsUrl("drs://my-fake-access-url2.com", "/path/to/somewhereelse2"),
+    UnresolvedDrsUrl("drs://my-fake-access-url3.com", "/path/to/somewhereelse3"),
+    UnresolvedDrsUrl("drs://my-fake-access-url4.com", "/path/to/somewhereelse4")
+  ))
+
   behavior of "DrsLocalizerMain"
 
   it should "fail if drs input is not passed" in {
@@ -29,291 +52,413 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     DrsLocalizerMain.run(List(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly)).unsafeRunSync() shouldBe ExitCode.Error
   }
 
-  it should "accept arguments and run successfully without Requester Pays ID" in {
-    val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None)
-    val expected = GcsUriDownloader(
-      gcsUrl = "gs://abc/foo-123/abc123",
-      serviceAccountJson = None,
-      downloadLoc = fakeDownloadLocation,
-      requesterPaysProjectIdOption = None)
-    mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-  }
-
-  it should "run successfully with all 3 arguments" in {
-    val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, Option(fakeRequesterPaysId))
-    val expected = GcsUriDownloader(
-      gcsUrl = "gs://abc/foo-123/abc123",
-      serviceAccountJson = None,
-      downloadLoc = fakeDownloadLocation,
-      requesterPaysProjectIdOption = Option(fakeRequesterPaysId))
-    mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-  }
-
-  it should "fail and throw error if the DRS Resolver response does not have gs:// url" in {
-    val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithoutAnyResolution, fakeDownloadLocation, None)
+  it should "tolerate no URLs being provided" in {
+    val mockDownloadFactory = new DownloaderFactory {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+        // This test path should never ask for the Google downloader
+        throw new RuntimeException("test failure111")
+      }
 
-    the[RuntimeException] thrownBy {
-      mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync()
-    } should have message "No access URL nor GCS URI starting with 'gs://' found in the DRS Resolver response!"
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+        // This test path should never ask for the Bulk downloader
+        throw new RuntimeException("test failure111")
+      }
+    }
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List()), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    downloaders.length shouldBe 0
   }
 
-  it should "resolve to use the correct downloader for an access url" in {
-    val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None)
-    val expected = AccessUrlDownloader(
-      accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None),
-      downloadLoc = fakeDownloadLocation,
-      hashes = FakeHashes
-    )
-    mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-  }
+  it should "build correct downloader(s) for a single google URL" in {
+    val mockDownloadFactory = new DownloaderFactory {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+        IO(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+      }
 
-  it should "resolve to use the correct downloader for an access url when the DRS Resolver response also contains a gs url" in {
-    val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution, fakeDownloadLocation, None)
-    val expected = AccessUrlDownloader(
-      accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None), downloadLoc = fakeDownloadLocation,
-      hashes = FakeHashes
-    )
-    mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-  }
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+        // This test path should never ask for the Bulk downloader
+        throw new RuntimeException("test failure111")
+      }
+    }
 
-  it should "not retry on access URL download success" in {
-    var actualAttempts = 0
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeGoogleUrls.head._1)), mockDownloadFactory,FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders :List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    downloaders.length shouldBe 1
 
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        super.resolveAndDownload(downloaderFactory)
-      }
+    val correct = downloaders.head match {
+      case _: GcsUriDownloader => true
+      case _ => false
     }
-    val accessUrlDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(DownloadSuccess)
-    })
+    correct shouldBe true
+  }
 
-    val downloaderFactory = new DownloaderFactory {
+  it should "build correct downloader(s) for a single access URL" in {
+    val mockDownloadFactory = new DownloaderFactory {
       override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+        IO(BulkAccessUrlDownloader(urlsToDownload))
       }
     }
 
-    drsLocalizer.resolveAndDownloadWithRetries(
-      downloadRetries = 3,
-      checksumRetries = 1,
-      downloaderFactory = downloaderFactory,
-      backoff = None
-    ).unsafeRunSync() shouldBe DownloadSuccess
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeAccessUrls.head._1)), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    downloaders.length shouldBe 1
 
-    actualAttempts shouldBe 1
+    val correct = downloaders.head match {
+      case _: BulkAccessUrlDownloader => true
+      case _ => false
+    }
+    correct shouldBe true
   }
 
-  it should "retry an appropriate number of times for regular retryable access URL download failures" in {
-    var actualAttempts = 0
+  it should "build correct downloader(s) for multiple google URLs" in {
+    val mockDownloadFactory = new DownloaderFactory {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+        IO(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+      }
 
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        super.resolveAndDownload(downloaderFactory)
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+        // This test path should never ask for the GCS downloader
+        throw new RuntimeException("test failure")
       }
     }
-    val accessUrlDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
+    val unresolvedUrls : List[UnresolvedDrsUrl] = fakeGoogleUrls.map(pair => pair._1).toList
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    downloaders.length shouldBe unresolvedUrls.length
+
+    val countGoogleDownloaders = downloaders.count(downloader => downloader match {
+      case _: GcsUriDownloader => true
+      case _ => false
     })
+    // We expect one GCS downloader for each GCS uri provided
+    countGoogleDownloaders shouldBe downloaders.length
+  }
 
-    val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        accessUrlDownloader
-      }
-
+  it should "build a single bulk downloader for multiple access URLs" in {
+    val mockDownloadFactory = new DownloaderFactory {
       override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation : Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+        IO(BulkAccessUrlDownloader(urlsToDownload))
       }
     }
+    val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    downloaders.length shouldBe 1
+
+    val countBulkDownloaders = downloaders.count(downloader => downloader match {
+      case _: BulkAccessUrlDownloader => true
+      case _ => false
+    })
+    // We expect one GCS downloader for each GCS uri provided
+    countBulkDownloaders shouldBe 1
+  }
 
-    assertThrows[Throwable] {
-      drsLocalizer.resolveAndDownloadWithRetries(
-        downloadRetries = 3,
-        checksumRetries = 1,
-        downloaderFactory = downloaderFactory,
-        backoff = None
-      ).unsafeRunSync()
-    }
+  it should "build 1 bulk downloader and 5 google downloaders for a mix of URLs" in {
+    val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList ++ fakeGoogleUrls.map(pair => pair._1).toList
+    val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+
+    downloaders.length shouldBe 6
 
-    actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
+    //we expect a single bulk downloader despite 5 access URLs being provided
+    val countBulkDownloaders = downloaders.count(downloader => downloader match {
+      case _: BulkAccessUrlDownloader => true
+      case _ => false
+    })
+    // We expect one GCS downloader for each GCS uri provided
+    countBulkDownloaders shouldBe 1
+    val countGoogleDownloaders = downloaders.count(downloader => downloader match {
+      case _: GcsUriDownloader => true
+      case _ => false
+    })
+    // We expect one GCS downloader for each GCS uri provided
+    countBulkDownloaders shouldBe 1
+    countGoogleDownloaders shouldBe 5
   }
 
-  it should "retry an appropriate number of times for fatal retryable access URL download failures" in {
-    var actualAttempts = 0
+  it should "accept arguments and run successfully without Requester Pays ID" in {
+    val unresolved = fakeGoogleUrls.head._1
+    val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, None)
+    val expected = GcsUriDownloader(
+      gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get,
+      serviceAccountJson = None,
+      downloadLoc = unresolved.downloadDestinationPath,
+      requesterPaysProjectIdOption = None)
+    val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head
+    downloader shouldBe expected
+  }
 
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        IO.raiseError(new RuntimeException("testing: fatal error") with FatalRetryDisposition)
-      }
+    it should "run successfully with all 3 arguments" in {
+      val unresolved = fakeGoogleUrls.head._1
+      val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+      val expected = GcsUriDownloader(
+        gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get,
+        serviceAccountJson = None,
+        downloadLoc = unresolved.downloadDestinationPath,
+        requesterPaysProjectIdOption = Option(fakeRequesterPaysId))
+      val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head
+      downloader shouldBe expected
     }
+/*
+    it should "fail and throw error if the DRS Resolver response does not have gs:// url" in {
+      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithoutAnyResolution, fakeDownloadLocation, None)
 
-    val accessUrlDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
-    })
+      the[RuntimeException] thrownBy {
+        mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync()
+      } should have message "No access URL nor GCS URI starting with 'gs://' found in the DRS Resolver response!"
+    }
 
-    val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        accessUrlDownloader
-      }
+    it should "resolve to use the correct downloader for an access url" in {
+      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None)
+      val expected = AccessUrlDownloader(
+        accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None),
+        downloadLoc = fakeDownloadLocation,
+        hashes = FakeHashes
+      )
+      mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
+    }
 
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
-      }
+    it should "resolve to use the correct downloader for an access url when the DRS Resolver response also contains a gs url" in {
+      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution, fakeDownloadLocation, None)
+      val expected = AccessUrlDownloader(
+        accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None), downloadLoc = fakeDownloadLocation,
+        hashes = FakeHashes
+      )
+      mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
+    }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
+    it should "not retry on access URL download success" in {
+      var actualAttempts = 0
+
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          super.resolveAndDownload(downloaderFactory)
+        }
+      }
+      val accessUrlDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(DownloadSuccess)
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          // This test path should never ask for the GCS downloader
+          throw new RuntimeException("test failure")
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
       }
-    }
 
-    assertThrows[Throwable] {
       drsLocalizer.resolveAndDownloadWithRetries(
         downloadRetries = 3,
         checksumRetries = 1,
         downloaderFactory = downloaderFactory,
         backoff = None
-      ).unsafeRunSync()
+      ).unsafeRunSync() shouldBe DownloadSuccess
+
+      actualAttempts shouldBe 1
     }
 
-    actualAttempts shouldBe 1 // 1 and done with a fatal exception
-  }
+    it should "retry an appropriate number of times for regular retryable access URL download failures" in {
+      var actualAttempts = 0
 
-  it should "not retry on GCS URI download success" in {
-    var actualAttempts = 0
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        super.resolveAndDownload(downloaderFactory)
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          super.resolveAndDownload(downloaderFactory)
+        }
       }
-    }
-    val gcsUriDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(DownloadSuccess)
-    })
-
-    val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        // This test path should never ask for the access URL downloader
-        throw new RuntimeException("test failure")
+      val accessUrlDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          // This test path should never ask for the GCS downloader
+          throw new RuntimeException("test failure")
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
       }
 
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        gcsUriDownloader
+      assertThrows[Throwable] {
+        drsLocalizer.resolveAndDownloadWithRetries(
+          downloadRetries = 3,
+          checksumRetries = 1,
+          downloaderFactory = downloaderFactory,
+          backoff = None
+        ).unsafeRunSync()
       }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
-      }
+      actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
     }
 
-    drsLocalizer.resolveAndDownloadWithRetries(
-      downloadRetries = 3,
-      checksumRetries = 1,
-      downloaderFactory = downloaderFactory,
-      backoff = None).unsafeRunSync()
+    it should "retry an appropriate number of times for fatal retryable access URL download failures" in {
+      var actualAttempts = 0
 
-    actualAttempts shouldBe 1
-  }
-
-  it should "retry an appropriate number of times for retryable GCS URI download failures" in {
-    var actualAttempts = 0
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        super.resolveAndDownload(downloaderFactory)
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          IO.raiseError(new RuntimeException("testing: fatal error") with FatalRetryDisposition)
+        }
       }
-    }
-    val gcsUriDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(1)))
-    })
 
-    val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        // This test path should never ask for the access URL downloader
-        throw new RuntimeException("test failure")
+      val accessUrlDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          // This test path should never ask for the GCS downloader
+          throw new RuntimeException("test failure")
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
       }
 
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        gcsUriDownloader
+      assertThrows[Throwable] {
+        drsLocalizer.resolveAndDownloadWithRetries(
+          downloadRetries = 3,
+          checksumRetries = 1,
+          downloaderFactory = downloaderFactory,
+          backoff = None
+        ).unsafeRunSync()
       }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
-      }
+      actualAttempts shouldBe 1 // 1 and done with a fatal exception
     }
 
-    assertThrows[Throwable] {
+    it should "not retry on GCS URI download success" in {
+      var actualAttempts = 0
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          super.resolveAndDownload(downloaderFactory)
+        }
+      }
+      val gcsUriDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(DownloadSuccess)
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          gcsUriDownloader
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
+      }
+
       drsLocalizer.resolveAndDownloadWithRetries(
         downloadRetries = 3,
         checksumRetries = 1,
         downloaderFactory = downloaderFactory,
         backoff = None).unsafeRunSync()
+
+      actualAttempts shouldBe 1
     }
 
-    actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
-  }
+    it should "retry an appropriate number of times for retryable GCS URI download failures" in {
+      var actualAttempts = 0
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          super.resolveAndDownload(downloaderFactory)
+        }
+      }
+      val gcsUriDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(1)))
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
+          // This test path should never ask for the access URL downloader
+          throw new RuntimeException("test failure")
+        }
+
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          gcsUriDownloader
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
+      }
 
-  it should "retry an appropriate number of times for checksum failures" in {
-    var actualAttempts = 0
-    val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-      override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-        actualAttempts = actualAttempts + 1
-        super.resolveAndDownload(downloaderFactory)
+      assertThrows[Throwable] {
+        drsLocalizer.resolveAndDownloadWithRetries(
+          downloadRetries = 3,
+          checksumRetries = 1,
+          downloaderFactory = downloaderFactory,
+          backoff = None).unsafeRunSync()
       }
+
+      actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
     }
-    val accessUrlDownloader = IO.pure(new Downloader {
-      override def download: IO[DownloadResult] =
-        IO.pure(ChecksumFailure)
-    })
 
-    val downloaderFactory = new DownloaderFactory {
-      override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-        accessUrlDownloader
+    it should "retry an appropriate number of times for checksum failures" in {
+      var actualAttempts = 0
+      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
+        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
+          actualAttempts = actualAttempts + 1
+          super.resolveAndDownload(downloaderFactory)
+        }
       }
-
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        // This test path should never ask for the GCS URI downloader.
-        throw new RuntimeException("test failure")
+      val accessUrlDownloader = IO.pure(new Downloader {
+        override def download: IO[DownloadResult] =
+          IO.pure(ChecksumFailure)
+      })
+
+      val downloaderFactory = new DownloaderFactory {
+        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+          // This test path should never ask for the GCS URI downloader.
+          throw new RuntimeException("test failure")
+        }
+
+        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+          // This test path should never ask for the Bulk downloader
+          throw new RuntimeException("test failure")
+        }
       }
 
-      override def buildBulkAccessUrlDownloader(urlToDownloadLocation: Map[AccessUrl, String]): IO[Downloader] = {
-        // This test path should never ask for the GCS downloader
-        throw new RuntimeException("test failure")
+      assertThrows[Throwable] {
+        drsLocalizer.resolveAndDownloadWithRetries(
+          downloadRetries = 3,
+          checksumRetries = 1,
+          downloaderFactory = downloaderFactory,
+          backoff = None).unsafeRunSync()
       }
-    }
 
-    assertThrows[Throwable] {
-      drsLocalizer.resolveAndDownloadWithRetries(
-        downloadRetries = 3,
-        checksumRetries = 1,
-        downloaderFactory = downloaderFactory,
-        backoff = None).unsafeRunSync()
+      actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
     }
-
-    actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
-  }
+    */
 }
 
 object MockDrsPaths {
@@ -321,21 +466,46 @@ object MockDrsPaths {
   val fakeDrsUrlWithAccessUrlResolutionOnly = "drs://def/bar-456/def456"
   val fakeDrsUrlWithAccessUrlAndGcsResolution = "drs://ghi/baz-789/ghi789"
   val fakeDrsUrlWithoutAnyResolution = "drs://foo/bar/no-gcs-path"
+
+  val fakeGoogleUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map(
+    (UnresolvedDrsUrl("drs://abc/foo-123/google/0", "/path/to/google/local0"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri0")), "/path/to/google/local0", URIType.GCS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/google/1", "/path/to/google/local1"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri1")), "/path/to/google/local1", URIType.GCS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/google/2", "/path/to/google/local2"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri2")), "/path/to/google/local2", URIType.GCS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/google/3", "/path/to/google/local3"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri3")), "/path/to/google/local3", URIType.GCS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/google/4", "/path/to/google/local4"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri4")), "/path/to/google/local4", URIType.GCS))
+  )
+
+  val fakeAccessUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map(
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), "/path/to/access/local0", URIType.HTTPS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), "/path/to/access/local1", URIType.HTTPS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), "/path/to/access/local2", URIType.HTTPS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), "/path/to/access/local3", URIType.HTTPS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), "/path/to/access/local4", URIType.HTTPS))
+  )
 }
 
 
-class MockDrsLocalizerMain(drsUrl: String,
-                           downloadLoc: String,
-                           requesterPaysProjectIdOption: Option[String],
+class MockDrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
+                           downloaderFactory: DownloaderFactory,
+                           drsCredentials: DrsCredentials,
+                           requesterPaysProjectIdOption: Option[String]
                           )
 
-  extends DrsLocalizerMain(drsUrl, downloadLoc, FakeAccessTokenStrategy, requesterPaysProjectIdOption) {
+  extends DrsLocalizerMain(toResolveAndDownload, downloaderFactory, FakeAccessTokenStrategy, requesterPaysProjectIdOption) {
 
   override def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
     IO {
       new MockDrsLocalizerDrsPathResolver(cloud.nio.impl.drs.MockDrsPaths.mockDrsConfig)
     }
   }
+  override def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
+    IO {
+      if (!fakeAccessUrls.contains(drsUrlToResolve) && !fakeGoogleUrls.contains(drsUrlToResolve)) {
+        throw new RuntimeException("Unexpected URI during testing")
+      }
+      fakeAccessUrls.getOrElse(drsUrlToResolve, fakeGoogleUrls.getOrElse(drsUrlToResolve, ResolvedDrsUrl(DrsResolverResponse(),"/12/3/", URIType.UNKNOWN)))
+    }
+  }
 }
 
 
@@ -343,6 +513,7 @@ class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends
   DrsLocalizerDrsPathResolver(drsConfig, FakeAccessTokenStrategy) {
 
   override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = {
+
     val drsResolverResponse = DrsResolverResponse(
       size = Option(1234),
       hashes = FakeHashes
@@ -371,5 +542,4 @@ object MockDrsLocalizerDrsPathResolver {
   val FakeAccessTokenStrategy: DrsCredentials = new DrsCredentials {
     override def getAccessToken: ErrorOr[String] = "testing code: do not call me".invalidNel
   }
-
 }

From c66ee6f1f59da5c805beb42f0f956f5084cd2e29 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 12 Sep 2023 14:46:26 -0400
Subject: [PATCH 24/36] more better tests

---
 .../drs/localizer/DrsLocalizerMain.scala      |  80 ++++++++-----
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 107 ++++++++----------
 .../BulkAccessUrlDownloaderSpec.scala         |   6 +-
 3 files changed, 104 insertions(+), 89 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 9b36152ac14..1205c46c0e3 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -7,7 +7,7 @@ import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
-import drs.localizer.DrsLocalizerMain.toUriType
+import drs.localizer.DrsLocalizerMain.toValidatedUriType
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
 
@@ -84,27 +84,50 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
     }
 
-  def toUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = {
+  /**
+    * Helper function to decide which downloader to use based on data from the DRS response.
+    * Throws a runtime exception if the DRS response is invalid.
+    */
+  def toValidatedUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = {
+    // if both are provided, prefer using access urls
     (accessUrl, gsUri) match {
+      case (Some(_), _) =>
+        if(!accessUrl.get.url.startsWith("https://")) { throw new RuntimeException("Resolved Access URL does not start with https://")}
+        URIType.ACCESS
       case (_, Some(_)) =>
+        if(!gsUri.get.startsWith("gs://")) { throw new RuntimeException("Resolved Google URL does not start with gs://")}
         URIType.GCS
-      case (Some(_), _) =>
-        URIType.HTTPS
       case (_, _) =>
-        URIType.UNKNOWN
+        throw new RuntimeException("DRS response did not contain any URLs")
     }
   }
   }
 
 object URIType extends Enumeration {
   type URIType = Value
-  val GCS, HTTPS, UNKNOWN = Value
+  val GCS, ACCESS, UNKNOWN = Value
 }
 
 class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
                        downloaderFactory: DownloaderFactory,
                        drsCredentials: DrsCredentials,
                        requesterPaysProjectIdOption: Option[String]) extends StrictLogging {
+
+  /**
+    * This will:
+    *   - resolve all URLS
+    *   - build downloader(s) for them
+    *   - Invoke the downloaders to localize the files.
+    * @return DownloadSuccess if all downloads succeed. An error otherwise.
+    */
+  def resolveAndDownload(): IO[DownloadResult] = {
+    IO {
+      val downloaders: List[Downloader] = buildDownloaders().unsafeRunSync()
+      val results: List[DownloadResult] = downloaders.map(downloader => downloader.download.unsafeRunSync())
+      results.find(res => res != DownloadSuccess).getOrElse(DownloadSuccess)
+    }
+  }
+
   def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = {
     IO {
       val drsConfig = DrsConfig.fromEnv(sys.env)
@@ -113,42 +136,41 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     }
   }
 
-  def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]) : IO[List[ResolvedDrsUrl]] = {
-    unresolvedUrls.flatMap{unresolvedList =>
-      getDrsPathResolver.flatMap{resolver =>
-        unresolvedList.map{unresolvedUrl =>
-          resolveSingleUrl(resolver, unresolvedUrl)
-        }.traverse(identity)
-      }
-    }
-  }
-
+  /**
+    * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver.
+    */
   def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
     IO {
       val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
       //Insert retry logic here.
       val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
-      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toUriType(drsResponse.accessUrl, drsResponse.gsUri))
+      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(drsResponse.accessUrl, drsResponse.gsUri))
     }
   }
 
-
-
-  def resolveAndDownload() : IO[DownloadResult] = {
-    IO {
-      val downloaders: List[Downloader] = buildDownloaders().unsafeRunSync()
-      val results: List[DownloadResult] = downloaders.map(downloader => downloader.download.unsafeRunSync())
-      results.find(res => res!= DownloadSuccess).getOrElse(DownloadSuccess)
+  /**
+    * Runs synchronous HTTP requests to resolve all the DRS urls.
+    */
+  def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]) : IO[List[ResolvedDrsUrl]] = {
+    unresolvedUrls.flatMap{unresolvedList =>
+      getDrsPathResolver.flatMap{resolver =>
+        unresolvedList.map{unresolvedUrl =>
+          resolveSingleUrl(resolver, unresolvedUrl)
+        }.traverse(identity)
+      }
     }
   }
+
+  /**
+    * After resolving all of the URLs, this sorts them into an "Access" or "GCS" bucket.
+    * All access URLS will be downloaded as a batch with a single bulk downloader.
+    * All google URLs will be downloaded individually in their own google downloader.
+    * @return List of all downloaders required to fulfill the request.
+    */
   def buildDownloaders() : IO[List[Downloader]] = {
     resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
-      val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.HTTPS)
+      val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.ACCESS)
       val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
-      pendingDownloads.filter(url => url.uriType == URIType.UNKNOWN).map(
-        unknown => logger.error(
-          s"Url is not an https:// or  gs:// URL: " +
-            s"${unknown.drsResponse.accessUrl.getOrElse(unknown.drsResponse.gsUri.getOrElse("missing_url"))}"))
       val bulkDownloader: Option[List[IO[Downloader]]] = if(accessUrls.isEmpty) None else Option(List(buildBulkAccessUrlDownloader(accessUrls)))
       val googleDownloaders: Option[List[IO[Downloader]]] = if(googleUrls.isEmpty) None else  Option(buildGoogleDownloaders(googleUrls))
       val combined: List[IO[Downloader]] = googleDownloaders.map(list => list).getOrElse(List()) ++ bulkDownloader.map(list => list).getOrElse(List())
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 4ded6ec2ff7..b5220d17033 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -108,11 +108,10 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
     downloaders.length shouldBe 1
 
-    val correct = downloaders.head match {
-      case _: BulkAccessUrlDownloader => true
-      case _ => false
-    }
-    correct shouldBe true
+    val expected = BulkAccessUrlDownloader(
+      List(fakeAccessUrls.head._2)
+    )
+    expected shouldEqual downloaders.head
   }
 
   it should "build correct downloader(s) for multiple google URLs" in {
@@ -159,8 +158,12 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       case _: BulkAccessUrlDownloader => true
       case _ => false
     })
-    // We expect one GCS downloader for each GCS uri provided
+    // We expect one total Bulk downloader for all access URIs to share
     countBulkDownloaders shouldBe 1
+    val expected = BulkAccessUrlDownloader(
+      fakeAccessUrls.map(pair => pair._2).toList
+    )
+    expected shouldEqual downloaders.head
   }
 
   it should "build 1 bulk downloader and 5 google downloaders for a mix of URLs" in {
@@ -198,45 +201,46 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     downloader shouldBe expected
   }
 
-    it should "run successfully with all 3 arguments" in {
-      val unresolved = fakeGoogleUrls.head._1
-      val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
-      val expected = GcsUriDownloader(
-        gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get,
-        serviceAccountJson = None,
-        downloadLoc = unresolved.downloadDestinationPath,
-        requesterPaysProjectIdOption = Option(fakeRequesterPaysId))
-      val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head
-      downloader shouldBe expected
-    }
-/*
-    it should "fail and throw error if the DRS Resolver response does not have gs:// url" in {
-      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithoutAnyResolution, fakeDownloadLocation, None)
+  it should "run successfully with all 3 arguments" in {
+    val unresolved = fakeGoogleUrls.head._1
+    val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
+    val expected = GcsUriDownloader(
+      gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get,
+      serviceAccountJson = None,
+      downloadLoc = unresolved.downloadDestinationPath,
+      requesterPaysProjectIdOption = Option(fakeRequesterPaysId))
+    val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head
+    downloader shouldBe expected
+  }
 
-      the[RuntimeException] thrownBy {
-        mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync()
-      } should have message "No access URL nor GCS URI starting with 'gs://' found in the DRS Resolver response!"
-    }
+  it should "successfully identify uri types, preferring access" in {
+    val exampleAccessResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes)))
+    val exampleGoogleResponse = DrsResolverResponse(gsUri = Option("gs://something"))
+    val exampleMixedResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes)), gsUri = Option("gs://something"))
+    DrsLocalizerMain.toValidatedUriType(exampleAccessResponse.accessUrl, exampleAccessResponse.gsUri) shouldBe URIType.ACCESS
+    DrsLocalizerMain.toValidatedUriType(exampleGoogleResponse.accessUrl, exampleGoogleResponse.gsUri) shouldBe URIType.GCS
+    DrsLocalizerMain.toValidatedUriType(exampleMixedResponse.accessUrl, exampleMixedResponse.gsUri) shouldBe URIType.ACCESS
+  }
 
-    it should "resolve to use the correct downloader for an access url" in {
-      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None)
-      val expected = AccessUrlDownloader(
-        accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None),
-        downloadLoc = fakeDownloadLocation,
-        hashes = FakeHashes
-      )
-      mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-    }
+  it should "throw an exception if the DRS Resolver response is invalid" in {
+    val badAccessResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("hQQps://something.com", FakeHashes)))
+    val badGoogleResponse = DrsResolverResponse(gsUri = Option("gQQs://something"))
+    val emptyResponse = DrsResolverResponse()
 
-    it should "resolve to use the correct downloader for an access url when the DRS Resolver response also contains a gs url" in {
-      val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution, fakeDownloadLocation, None)
-      val expected = AccessUrlDownloader(
-        accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None), downloadLoc = fakeDownloadLocation,
-        hashes = FakeHashes
-      )
-      mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected
-    }
+    the[RuntimeException] thrownBy {
+      DrsLocalizerMain.toValidatedUriType(badAccessResponse.accessUrl, badAccessResponse.gsUri)
+    } should have message "Resolved Access URL does not start with https://"
 
+    the[RuntimeException] thrownBy {
+      DrsLocalizerMain.toValidatedUriType(badGoogleResponse.accessUrl, badGoogleResponse.gsUri)
+    } should have message "Resolved Google URL does not start with gs://"
+
+    the[RuntimeException] thrownBy {
+      DrsLocalizerMain.toValidatedUriType(emptyResponse.accessUrl, emptyResponse.gsUri)
+    } should have message "DRS response did not contain any URLs"
+  }
+
+/*
     it should "not retry on access URL download success" in {
       var actualAttempts = 0
 
@@ -397,11 +401,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       })
 
       val downloaderFactory = new DownloaderFactory {
-        override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = {
-          // This test path should never ask for the access URL downloader
-          throw new RuntimeException("test failure")
-        }
-
         override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
           gcsUriDownloader
         }
@@ -431,10 +430,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
           super.resolveAndDownload(downloaderFactory)
         }
       }
-      val accessUrlDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(ChecksumFailure)
-      })
 
       val downloaderFactory = new DownloaderFactory {
         override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
@@ -455,7 +450,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
           downloaderFactory = downloaderFactory,
           backoff = None).unsafeRunSync()
       }
-
       actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
     }
     */
@@ -476,11 +470,11 @@ object MockDrsPaths {
   )
 
   val fakeAccessUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map(
-    (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), "/path/to/access/local0", URIType.HTTPS)),
-    (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), "/path/to/access/local1", URIType.HTTPS)),
-    (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), "/path/to/access/local2", URIType.HTTPS)),
-    (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), "/path/to/access/local3", URIType.HTTPS)),
-    (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), "/path/to/access/local4", URIType.HTTPS))
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), "/path/to/access/local0", URIType.ACCESS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), "/path/to/access/local1", URIType.ACCESS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), "/path/to/access/local2", URIType.ACCESS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), "/path/to/access/local3", URIType.ACCESS)),
+    (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), "/path/to/access/local4", URIType.ACCESS))
   )
 }
 
@@ -508,7 +502,6 @@ class MockDrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
   }
 }
 
-
 class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends
   DrsLocalizerDrsPathResolver(drsConfig, FakeAccessTokenStrategy) {
 
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index 20b5e4c8aa0..491aa88cc0b 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -9,9 +9,9 @@ import org.scalatest.matchers.should.Matchers
 import java.nio.file.Path
 
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
-  val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.HTTPS)
-  val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), "path/to/local/download/dest2", URIType.HTTPS)
-  val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), "path/to/local/download/dest3", URIType.HTTPS)
+  val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.ACCESS)
+  val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), "path/to/local/download/dest2", URIType.ACCESS)
+  val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), "path/to/local/download/dest3", URIType.ACCESS)
   val emptyList : List[ResolvedDrsUrl] = List()
   val oneElement: List[ResolvedDrsUrl] = List(ex1)
   val threeElements: List[ResolvedDrsUrl] = List(ex1, ex2, ex3)

From 2a4ae0d664413d547e65af2511ebb31f0d48647d Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 12 Sep 2023 15:21:03 -0400
Subject: [PATCH 25/36] fix hashing

---
 .../drs/localizer/downloaders/BulkAccessUrlDownloader.scala  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index b5b50909c7d..29e59ad19eb 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -43,8 +43,8 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
       //NB: trailing comma is being removed in generateJsonManifest
       val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse(AccessUrl("missing", None))
       drsResponse.hashes.map(_ => {
-        val checksum = GetmChecksum(drsResponse.hashes, accessUrl)
-        val checksumAlgorithm = checksum.getmAlgorithm
+        val checksum = GetmChecksum(drsResponse.hashes, accessUrl).value.getOrElse("error_calculating_checksum")
+        val checksumAlgorithm = GetmChecksum(drsResponse.hashes, accessUrl).getmAlgorithm
         s"""  {
            |    "url" : "${accessUrl.url}",
            |    "filepath" : "$destinationFilepath",
@@ -70,6 +70,7 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
         jsonString = jsonString.substring(0, jsonString.lastIndexOf(","))
       }
       jsonString += "\n]"
+      logger.info(jsonString)
       Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
     }
   }

From cb4fb5d24a910f73bf8b4312193b1d668f11495a Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Tue, 12 Sep 2023 16:51:17 -0400
Subject: [PATCH 26/36] who doesnt love a good implicit actor system

---
 build.sbt                                     |  1 +
 .../drs/localizer/DrsLocalizerMain.scala      | 30 +++++++++++++++----
 .../downloaders/BulkAccessUrlDownloader.scala |  4 ---
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/build.sbt b/build.sbt
index 2c9a8068992..4d890832fd4 100644
--- a/build.sbt
+++ b/build.sbt
@@ -396,6 +396,7 @@ lazy val `cromwell-drs-localizer` = project
   .dependsOn(`cloud-nio-impl-drs`)
   .dependsOn(common)
   .dependsOn(`cloud-nio-impl-drs` % "test->test")
+  .dependsOn(core)
 
 lazy val pact4s = project.in(file("pact4s"))
   .settings(pact4sSettings)
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 1205c46c0e3..45867036627 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -10,13 +10,17 @@ import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
 import drs.localizer.DrsLocalizerMain.toValidatedUriType
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
-
+import cromwell.core.retry.Retry.withRetry
+import cromwell.core.retry.SimpleExponentialBackoff
+import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Props}
 import java.io.File
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import drs.localizer.URIType.URIType
+
+import scala.concurrent.Future
 case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
 case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
@@ -136,15 +140,31 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     }
   }
 
+  /**
+    * Helper method used to retry failed HTTP requests.
+    * @param resolverObject
+    * @param drsUrlToResolve
+    * @return
+    */
+  def resolveDrsUrlOrFail(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl) : Future[ResolvedDrsUrl] = {
+    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+    val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
+    if(drsResponse.accessUrl.isEmpty && drsResponse.gsUri.isEmpty){
+      Future.failed(new RuntimeException("Response from DRS did not return a valid URL."))
+    } else {
+      Future.successful(ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(drsResponse.accessUrl, drsResponse.gsUri)))
+    }
+  }
+
   /**
     * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver.
     */
   def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
     IO {
-      val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-      //Insert retry logic here.
-      val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
-      ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(drsResponse.accessUrl, drsResponse.gsUri))
+      withRetry(
+        () => resolveDrsUrlOrFail(resolverObject, drsUrlToResolve),
+        maxRetries = Option(5)
+      )
     }
   }
 
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index 29e59ad19eb..2fcf7fca1cf 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -10,7 +10,6 @@ import scala.sys.process.{Process, ProcessLogger}
 import scala.util.matching.Regex
 import drs.localizer.ResolvedDrsUrl
 case class GetmResult(returnCode: Int, stderr: String)
-
 /**
   * Getm is a python tool that is used to download resolved DRS uris quickly and in parallel.
   * This class builds a getm-manifest.json file that it uses for input, and builds/executes a shell command
@@ -80,9 +79,6 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
   }
   def runGetm: IO[GetmResult] = {
     generateJsonManifest(resolvedUrls).flatMap{ manifestPath =>
-      //val script = s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm --manifest '$manifestPath'""" //TODO: Check if getm will automatically create directories, or if we need to do it for each file.
-      // also consider deleting files already there to make retires a little simpler?
-
       val script = generateGetmCommand(manifestPath)
       val copyCommand : Seq[String] = Seq("bash", "-c", script)
       logger.info(script)

From d32613f72f2b6aaf23bc1d6a1115d725c670da41 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Wed, 13 Sep 2023 08:11:34 -0400
Subject: [PATCH 27/36] low hanging feedback

---
 build.sbt                                     |  1 -
 .../drs/localizer/DrsLocalizerMain.scala      | 38 ++++---------------
 2 files changed, 7 insertions(+), 32 deletions(-)

diff --git a/build.sbt b/build.sbt
index 4d890832fd4..2c9a8068992 100644
--- a/build.sbt
+++ b/build.sbt
@@ -396,7 +396,6 @@ lazy val `cromwell-drs-localizer` = project
   .dependsOn(`cloud-nio-impl-drs`)
   .dependsOn(common)
   .dependsOn(`cloud-nio-impl-drs` % "test->test")
-  .dependsOn(core)
 
 lazy val pact4s = project.in(file("pact4s"))
   .settings(pact4sSettings)
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 45867036627..e713d843783 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -10,17 +10,12 @@ import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
 import drs.localizer.DrsLocalizerMain.toValidatedUriType
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
-import cromwell.core.retry.Retry.withRetry
-import cromwell.core.retry.SimpleExponentialBackoff
-import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Props}
 import java.io.File
 import java.nio.charset.Charset
 import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import drs.localizer.URIType.URIType
-
-import scala.concurrent.Future
 case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
 case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
@@ -70,6 +65,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       val openFile = new File(csvManifestPath)
       val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
       val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
+      csvParser.close()
       list
     }
   }
@@ -82,10 +78,8 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
       case None =>
         IO.pure(List(UnresolvedDrsUrl(commandLineArguments.drsObject.get, commandLineArguments.containerPath.get)))
       }
-    IO{
-      val main = new DrsLocalizerMain(urlList, defaultDownloaderFactory, drsCredentials, commandLineArguments.googleRequesterPaysProject)
-      main.resolveAndDownload().unsafeRunSync().exitCode
-    }
+    val main = new DrsLocalizerMain(urlList, defaultDownloaderFactory, drsCredentials, commandLineArguments.googleRequesterPaysProject)
+    main.resolveAndDownload().map(_.exitCode)
     }
 
   /**
@@ -140,32 +134,14 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     }
   }
 
-  /**
-    * Helper method used to retry failed HTTP requests.
-    * @param resolverObject
-    * @param drsUrlToResolve
-    * @return
-    */
-  def resolveDrsUrlOrFail(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl) : Future[ResolvedDrsUrl] = {
-    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-    val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields).unsafeRunSync()
-    if(drsResponse.accessUrl.isEmpty && drsResponse.gsUri.isEmpty){
-      Future.failed(new RuntimeException("Response from DRS did not return a valid URL."))
-    } else {
-      Future.successful(ResolvedDrsUrl(drsResponse, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(drsResponse.accessUrl, drsResponse.gsUri)))
-    }
-  }
-
   /**
     * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver.
     */
   def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
-    IO {
-      withRetry(
-        () => resolveDrsUrlOrFail(resolverObject, drsUrlToResolve),
-        maxRetries = Option(5)
-      )
-    }
+    //TODO: Add some retry logic here.
+    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+    val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields)
+    drsResponse.map(resp => ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri)))
   }
 
   /**

From 52c7055990d28a4dda16d72be9ace97d5969396c Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Wed, 13 Sep 2023 08:40:51 -0400
Subject: [PATCH 28/36] remove unnecessary IO

---
 .../drs/localizer/DrsLocalizerMain.scala      | 28 ++++++++---------
 .../downloaders/DownloaderFactory.scala       |  5 ++--
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 30 +++++++++----------
 3 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index e713d843783..c65add1cb7a 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -42,11 +42,11 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
 
   val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory {
-    override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] =
-      IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+    override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader =
+      GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption)
 
-    override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-      IO.pure(BulkAccessUrlDownloader(urlsToDownload))
+    override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
+      BulkAccessUrlDownloader(urlsToDownload)
     }
   }
 
@@ -119,10 +119,11 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     * @return DownloadSuccess if all downloads succeed. An error otherwise.
     */
   def resolveAndDownload(): IO[DownloadResult] = {
-    IO {
-      val downloaders: List[Downloader] = buildDownloaders().unsafeRunSync()
-      val results: List[DownloadResult] = downloaders.map(downloader => downloader.download.unsafeRunSync())
-      results.find(res => res != DownloadSuccess).getOrElse(DownloadSuccess)
+    val downloadResults = buildDownloaders().flatMap { downloaderList =>
+      downloaderList.map(downloader => downloader.download).traverse(identity)
+    }
+    downloadResults.map{list =>
+      list.find(result => result != DownloadSuccess).getOrElse(DownloadSuccess)
     }
   }
 
@@ -167,14 +168,13 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
       val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.ACCESS)
       val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
-      val bulkDownloader: Option[List[IO[Downloader]]] = if(accessUrls.isEmpty) None else Option(List(buildBulkAccessUrlDownloader(accessUrls)))
-      val googleDownloaders: Option[List[IO[Downloader]]] = if(googleUrls.isEmpty) None else  Option(buildGoogleDownloaders(googleUrls))
-      val combined: List[IO[Downloader]] = googleDownloaders.map(list => list).getOrElse(List()) ++ bulkDownloader.map(list => list).getOrElse(List())
-      combined.traverse(identity)
+      val bulkDownloader: Option[List[Downloader]] = if (accessUrls.isEmpty) None else Option(List(buildBulkAccessUrlDownloader(accessUrls)))
+      val googleDownloaders: Option[List[Downloader]] = if (googleUrls.isEmpty) None else Option(buildGoogleDownloaders(googleUrls))
+      IO.pure(googleDownloaders.map(list => list).getOrElse(List()) ++ bulkDownloader.map(list => list).getOrElse(List()))
     }
   }
 
-  def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]) : List[IO[Downloader]] = {
+  def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]) : List[Downloader] = {
     resolvedGoogleUrls.map{url=>
       downloaderFactory.buildGcsUriDownloader(
         gcsPath = url.drsResponse.gsUri.get,
@@ -183,7 +183,7 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
         requesterPaysProjectOption = requesterPaysProjectIdOption)
     }
   }
-  def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) : IO[Downloader] = {
+  def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) : Downloader = {
     downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls)
   }
 
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
index c7caae74ec7..6c7f27e8a6e 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala
@@ -1,13 +1,12 @@
 package drs.localizer.downloaders
 
-import cats.effect.IO
 import drs.localizer.ResolvedDrsUrl
 
 trait DownloaderFactory {
-  def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]) : IO[Downloader]
+  def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]) : Downloader
 
   def buildGcsUriDownloader(gcsPath: String,
                             serviceAccountJsonOption: Option[String],
                             downloadLoc: String,
-                            requesterPaysProjectOption: Option[String]): IO[Downloader]
+                            requesterPaysProjectOption: Option[String]): Downloader
 }
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index b5220d17033..07db22423a8 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -54,12 +54,12 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
   it should "tolerate no URLs being provided" in {
     val mockDownloadFactory = new DownloaderFactory {
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = {
         // This test path should never ask for the Google downloader
         throw new RuntimeException("test failure111")
       }
 
-      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
         // This test path should never ask for the Bulk downloader
         throw new RuntimeException("test failure111")
       }
@@ -71,18 +71,18 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
   it should "build correct downloader(s) for a single google URL" in {
     val mockDownloadFactory = new DownloaderFactory {
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        IO(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = {
+        GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption)
       }
 
-      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
         // This test path should never ask for the Bulk downloader
         throw new RuntimeException("test failure111")
       }
     }
 
     val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeGoogleUrls.head._1)), mockDownloadFactory,FakeAccessTokenStrategy, Option(fakeRequesterPaysId))
-    val downloaders :List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
+    val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync()
     downloaders.length shouldBe 1
 
     val correct = downloaders.head match {
@@ -94,13 +94,13 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
   it should "build correct downloader(s) for a single access URL" in {
     val mockDownloadFactory = new DownloaderFactory {
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
 
-      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-        IO(BulkAccessUrlDownloader(urlsToDownload))
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
+        BulkAccessUrlDownloader(urlsToDownload)
       }
     }
 
@@ -116,11 +116,11 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
   it should "build correct downloader(s) for multiple google URLs" in {
     val mockDownloadFactory = new DownloaderFactory {
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-        IO(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption))
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = {
+        GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption)
       }
 
-      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
@@ -140,13 +140,13 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
   it should "build a single bulk downloader for multiple access URLs" in {
     val mockDownloadFactory = new DownloaderFactory {
-      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
+      override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = {
         // This test path should never ask for the GCS downloader
         throw new RuntimeException("test failure")
       }
 
-      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-        IO(BulkAccessUrlDownloader(urlsToDownload))
+      override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = {
+        BulkAccessUrlDownloader(urlsToDownload)
       }
     }
     val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList

From 33670ebd58d7733d9c6164d1689348a4f0cd4147 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Wed, 13 Sep 2023 08:47:09 -0400
Subject: [PATCH 29/36] stale comment

---
 .../src/main/scala/drs/localizer/DrsLocalizerMain.scala       | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index c65add1cb7a..3260c883eea 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -56,8 +56,7 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
   }
 
   /**
-    * Helper function to read a CSV file as a map from drs URL to requested download destination
-    *
+    * Helper function to read a CSV file as pairs of drsURL -> local download destination.
     * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory
     */
   def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = {
@@ -70,7 +69,6 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     }
   }
 
-
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials) : IO[ExitCode] = {
     val urlList : IO[List[UnresolvedDrsUrl]] = commandLineArguments.manifestPath match {
       case Some(manifestPath) =>

From d745e2b75aadf990d30923bc4edf088ce3be7671 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Wed, 13 Sep 2023 14:27:57 -0400
Subject: [PATCH 30/36] teamwork

---
 .../drs/localizer/DrsLocalizerMain.scala      | 120 ++++++++++++------
 1 file changed, 78 insertions(+), 42 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 3260c883eea..09c86f83e85 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -3,6 +3,7 @@ package drs.localizer
 import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO, IOApp}
 import cats.implicits.toTraverseOps
+import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition}
 import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
@@ -16,6 +17,7 @@ import scala.concurrent.duration._
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import drs.localizer.URIType.URIType
+
 case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String)
 case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType)
 object DrsLocalizerMain extends IOApp with StrictLogging {
@@ -63,19 +65,26 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
     IO {
       val openFile = new File(csvManifestPath)
       val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT)
-      val list = csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
-      csvParser.close()
-      list
+      try{
+        csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList
+      } finally {
+        csvParser.close()
+      }
     }
   }
 
   def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials) : IO[ExitCode] = {
-    val urlList : IO[List[UnresolvedDrsUrl]] = commandLineArguments.manifestPath match {
-      case Some(manifestPath) =>
+    val urlList = (commandLineArguments.manifestPath, commandLineArguments.drsObject, commandLineArguments.containerPath) match {
+      case (Some(manifestPath), _, _) => {
         loadCSVManifest(manifestPath)
-      case None =>
-        IO.pure(List(UnresolvedDrsUrl(commandLineArguments.drsObject.get, commandLineArguments.containerPath.get)))
       }
+      case (_, Some(drsObject), Some(containerPath)) => {
+        IO.pure(List(UnresolvedDrsUrl(drsObject, containerPath)))
+      }
+      case(_,_,_) => {
+        throw new RuntimeException("Illegal command line arguments supplied to drs localizer.")
+      }
+    }
     val main = new DrsLocalizerMain(urlList, defaultDownloaderFactory, drsCredentials, commandLineArguments.googleRequesterPaysProject)
     main.resolveAndDownload().map(_.exitCode)
     }
@@ -133,29 +142,6 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     }
   }
 
-  /**
-    * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver.
-    */
-  def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
-    //TODO: Add some retry logic here.
-    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
-    val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields)
-    drsResponse.map(resp => ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri)))
-  }
-
-  /**
-    * Runs synchronous HTTP requests to resolve all the DRS urls.
-    */
-  def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]) : IO[List[ResolvedDrsUrl]] = {
-    unresolvedUrls.flatMap{unresolvedList =>
-      getDrsPathResolver.flatMap{resolver =>
-        unresolvedList.map{unresolvedUrl =>
-          resolveSingleUrl(resolver, unresolvedUrl)
-        }.traverse(identity)
-      }
-    }
-  }
-
   /**
     * After resolving all of the URLs, this sorts them into an "Access" or "GCS" bucket.
     * All access URLS will be downloaded as a batch with a single bulk downloader.
@@ -163,12 +149,12 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     * @return List of all downloaders required to fulfill the request.
     */
   def buildDownloaders() : IO[List[Downloader]] = {
-    resolveUrls(toResolveAndDownload).flatMap { pendingDownloads =>
+    resolveUrls(toResolveAndDownload).map { pendingDownloads =>
       val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.ACCESS)
       val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS)
-      val bulkDownloader: Option[List[Downloader]] = if (accessUrls.isEmpty) None else Option(List(buildBulkAccessUrlDownloader(accessUrls)))
-      val googleDownloaders: Option[List[Downloader]] = if (googleUrls.isEmpty) None else Option(buildGoogleDownloaders(googleUrls))
-      IO.pure(googleDownloaders.map(list => list).getOrElse(List()) ++ bulkDownloader.map(list => list).getOrElse(List()))
+      val bulkDownloader: List[Downloader] = if (accessUrls.isEmpty) List() else List(buildBulkAccessUrlDownloader(accessUrls))
+      val googleDownloaders: List[Downloader] = if (googleUrls.isEmpty) List() else buildGoogleDownloaders(googleUrls)
+      bulkDownloader ++ googleDownloaders
     }
   }
 
@@ -185,8 +171,60 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls)
   }
 
+  /**
+    * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver.
+    */
+  def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = {
+    val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes)
+    val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields)
+    drsResponse.map(resp => ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri)))
+  }
+
+
+  val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff(
+    initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
+
+  /**
+    * Runs synchronous HTTP requests to resolve all the DRS urls.
+    */
+  def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]): IO[List[ResolvedDrsUrl]] = {
+    unresolvedUrls.flatMap { unresolvedList =>
+      getDrsPathResolver.flatMap { resolver =>
+        unresolvedList.map { unresolvedUrl =>
+          resolveWithRetries(resolver, unresolvedUrl, 5, Option(defaultBackoff))
+        }.traverse(identity)
+      }
+    }
+  }
+
+  def resolveWithRetries(resolverObject: DrsLocalizerDrsPathResolver,
+                         drsUrlToResolve: UnresolvedDrsUrl,
+                         resolutionRetries: Int,
+                         backoff: Option[CloudNioBackoff],
+                        resolutionAttempt: Int = 0) : IO[ResolvedDrsUrl] = {
+
+    def maybeRetryForResolutionFailure(t: Throwable): IO[ResolvedDrsUrl] = {
+      if (resolutionAttempt < resolutionRetries) {
+        backoff foreach { b => Thread.sleep(b.backoffMillis) }
+        logger.warn(s"Attempting retry $resolutionAttempt of $resolutionRetries drs resolution retries to resolve ${drsUrlToResolve.drsUrl}", t)
+        resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff map { _.next }, resolutionAttempt+1)
+      } else {
+        IO.raiseError(new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve $drsUrlToResolve.drsUrl", t))
+      }
+    }
+
+    resolveSingleUrl(resolverObject, drsUrlToResolve).redeemWith(
+      recover = maybeRetryForResolutionFailure,
+      bind = {
+      case f: FatalRetryDisposition =>
+        IO.raiseError(new RuntimeException(s"Fatal error resolving DRS URL: $f"))
+      case _: RegularRetryDisposition =>
+        resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff, resolutionAttempt+1)
+      case o => IO.pure(o)
+    })
+  }
 
-  /*
+/*
   def resolveAndDownloadWithRetries(downloadRetries: Int,
                                     checksumRetries: Int,
                                     downloaderFactory: DownloaderFactory,
@@ -204,8 +242,7 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
         IO.raiseError(new RuntimeException(s"Exhausted $checksumRetries checksum retries to resolve, download and checksum $drsUrl", t))
       }
     }
-*/
-    /*
+
     def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = {
       t match {
         case _: FatalRetryDisposition =>
@@ -218,8 +255,7 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
           IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
       }
     }
-*/
-      /*
+
     resolveAndDownload(downloaderFactory).redeemWith({
       maybeRetryForDownloadFailure
     },
@@ -232,11 +268,11 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
       case ChecksumFailure =>
         maybeRetryForChecksumFailure(new RuntimeException(s"Checksum failure for $drsUrl on checksum retry attempt $checksumAttempt of $checksumRetries"))
       case o => IO.pure(o)
-    }*
-
+    }
 
     IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
   }
-         */
+
+ */
 }
 

From 3e1dec2ec918dac767ea3f76285cf849a21819b1 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 14 Sep 2023 07:39:41 -0400
Subject: [PATCH 31/36] cleanup file when done

---
 .../localizer/downloaders/BulkAccessUrlDownloader.scala  | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index 2fcf7fca1cf..6858a6956cb 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -17,6 +17,8 @@ case class GetmResult(returnCode: Int, stderr: String)
   * @param resolvedUrls
   */
 case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends Downloader with StrictLogging {
+
+  val getmManifestPath: Path = Paths.get("getm-manifest.json")
   /**
     * Write a json manifest to disk that looks like:
     * // [
@@ -70,10 +72,14 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
       }
       jsonString += "\n]"
       logger.info(jsonString)
-      Files.write(Paths.get("getm-manifest.json"), jsonString.getBytes(StandardCharsets.UTF_8))
+      Files.write(getmManifestPath, jsonString.getBytes(StandardCharsets.UTF_8))
     }
   }
 
+  def deleteJsonManifest() = {
+    Files.deleteIfExists(getmManifestPath)
+  }
+
   def generateGetmCommand(pathToMainfestJson : Path) : String = {
     s"""getm --manifest ${pathToMainfestJson.toString}"""
   }
@@ -86,6 +92,7 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
       val stderr = new StringBuilder()
       val errorCapture: String => Unit = { s => stderr.append(s); () }
       val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture)
+      deleteJsonManifest()
       logger.info(stderr.toString().trim())
       IO(GetmResult(returnCode, stderr.toString().trim()))
     }

From f6928d7c17e0e17596473696457cbae7fbaaaca1 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 14 Sep 2023 08:04:18 -0400
Subject: [PATCH 32/36] google download retries

---
 .../drs/localizer/DrsLocalizerMain.scala      | 51 -------------------
 .../downloaders/GcsUriDownloader.scala        | 41 ++++++++++++++-
 2 files changed, 40 insertions(+), 52 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 09c86f83e85..6148c53dafd 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -223,56 +223,5 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
       case o => IO.pure(o)
     })
   }
-
-/*
-  def resolveAndDownloadWithRetries(downloadRetries: Int,
-                                    checksumRetries: Int,
-                                    downloaderFactory: DownloaderFactory,
-                                    backoff: Option[CloudNioBackoff],
-                                    downloadAttempt: Int = 0,
-                                    checksumAttempt: Int = 0): IO[DownloadResult] = {
-
-    def maybeRetryForChecksumFailure(t: Throwable): IO[DownloadResult] = {
-      if (checksumAttempt < checksumRetries) {
-        backoff foreach { b => Thread.sleep(b.backoffMillis) }
-        logger.warn(s"Attempting retry $checksumAttempt of $checksumRetries checksum retries to download $drsUrl", t)
-        // In the event of a checksum failure reset the download attempt to zero.
-        resolveAndDownloadWithRetries(downloadRetries, checksumRetries, downloaderFactory, backoff map { _.next }, 0, checksumAttempt + 1)
-      } else {
-        IO.raiseError(new RuntimeException(s"Exhausted $checksumRetries checksum retries to resolve, download and checksum $drsUrl", t))
-      }
-    }
-
-    def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = {
-      t match {
-        case _: FatalRetryDisposition =>
-          IO.raiseError(t)
-        case _ if downloadAttempt < downloadRetries =>
-          backoff foreach { b => Thread.sleep(b.backoffMillis) }
-          logger.warn(s"Attempting retry $downloadAttempt of $downloadRetries download retries to download $drsUrl", t)
-          resolveAndDownloadWithRetries(downloadRetries, checksumRetries, downloaderFactory, backoff map { _.next }, downloadAttempt + 1, checksumAttempt)
-        case _ =>
-          IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
-      }
-    }
-
-    resolveAndDownload(downloaderFactory).redeemWith({
-      maybeRetryForDownloadFailure
-    },
-    {
-      case f: FatalDownloadFailure =>
-        IO.raiseError(new RuntimeException(s"Fatal error downloading DRS object: $f"))
-      case r: RetryableDownloadFailure =>
-        maybeRetryForDownloadFailure(
-          new RuntimeException(s"Retryable download error: $r for $drsUrl on retry attempt $downloadAttempt of $downloadRetries") with RegularRetryDisposition)
-      case ChecksumFailure =>
-        maybeRetryForChecksumFailure(new RuntimeException(s"Checksum failure for $drsUrl on checksum retry attempt $checksumAttempt of $checksumRetries"))
-      case o => IO.pure(o)
-    }
-
-    IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t))
-  }
-
- */
 }
 
diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
index d4c81af6300..3d82c54595f 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
@@ -1,10 +1,12 @@
 package drs.localizer.downloaders
 import cats.effect.{ExitCode, IO}
+import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.downloaders.GcsUriDownloader.RequesterPaysErrorMsg
-
+import scala.language.postfixOps
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path}
+import scala.concurrent.duration.DurationInt
 import scala.sys.process.{Process, ProcessLogger}
 
 case class GcsUriDownloader(gcsUrl: String,
@@ -12,7 +14,13 @@ case class GcsUriDownloader(gcsUrl: String,
                             downloadLoc: String,
                             requesterPaysProjectIdOption: Option[String]) extends Downloader with StrictLogging {
 
+  val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff(
+    initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
   override def download: IO[DownloadResult] = {
+    downloadWithRetries(5, Option(defaultBackoff))
+  }
+
+  def runDownloadCommand: IO[DownloadResult] = {
 
     logger.info(s"Requester Pays project ID is $requesterPaysProjectIdOption")
     logger.info(s"Attempting to download $gcsUrl to $downloadLoc")
@@ -40,6 +48,37 @@ case class GcsUriDownloader(gcsUrl: String,
     IO.pure(result)
   }
 
+  def downloadWithRetries(downloadRetries: Int,
+                          backoff: Option[CloudNioBackoff],
+                          downloadAttempt: Int = 0): IO[DownloadResult] =
+  {
+
+    def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = {
+      if (downloadAttempt < downloadRetries) {
+        backoff foreach { b => Thread.sleep(b.backoffMillis) }
+        logger.warn(s"Attempting download retry $downloadAttempt of $downloadRetries for a GCS url", t)
+        downloadWithRetries(downloadRetries, backoff map {
+          _.next
+        }, downloadAttempt + 1)
+      } else {
+        IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries resolution retries to download GCS file", t))
+      }
+    }
+
+    runDownloadCommand.redeemWith(
+      recover = maybeRetryForDownloadFailure,
+      bind = {
+        case s: DownloadSuccess.type =>
+          IO.pure(s)
+        case _: RecognizedRetryableDownloadFailure =>
+          downloadWithRetries(downloadRetries, backoff, downloadAttempt+1)
+        case _: UnrecognizedRetryableDownloadFailure =>
+          downloadWithRetries(downloadRetries, backoff, downloadAttempt+1)
+        case _ =>
+          downloadWithRetries(downloadRetries, backoff, downloadAttempt+1)
+      })
+  }
+
   /**
     * Bash to download the GCS file using `gsutil`.
     */

From 7e3fb8f8d5d8433aeeca00d783965de62b3316cf Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 14 Sep 2023 08:19:03 -0400
Subject: [PATCH 33/36] remove debugging log

---
 .../drs/localizer/downloaders/BulkAccessUrlDownloader.scala      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
index 6858a6956cb..4668c5072ed 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala
@@ -71,7 +71,6 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends
         jsonString = jsonString.substring(0, jsonString.lastIndexOf(","))
       }
       jsonString += "\n]"
-      logger.info(jsonString)
       Files.write(getmManifestPath, jsonString.getBytes(StandardCharsets.UTF_8))
     }
   }

From 256b8d1bf3ea711155b5715e0170e8580c5890f6 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Thu, 14 Sep 2023 08:43:29 -0400
Subject: [PATCH 34/36] migrate last test

---
 .../downloaders/AccessUrlDownloaderSpec.scala | 60 -------------------
 .../BulkAccessUrlDownloaderSpec.scala         | 37 +++++++++++-
 2 files changed, 36 insertions(+), 61 deletions(-)
 delete mode 100644 cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala

diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala
deleted file mode 100644
index f820ed9cae2..00000000000
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-package drs.localizer.downloaders
-/*
-import cats.effect.ExitCode
-import cats.syntax.validated._
-import cloud.nio.impl.drs.AccessUrl
-import common.assertion.CromwellTimeoutSpec
-import org.scalatest.flatspec.AnyFlatSpec
-import org.scalatest.matchers.should.Matchers
-import org.scalatest.prop.TableDrivenPropertyChecks._
-//TODO: Migrate to Bulk Tests
-class AccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
-  it should "return the correct download script for a url-only access URL, no requester pays" in {
-    val fakeDownloadLocation = "/root/foo/foo-123.bam"
-    val fakeAccessUrl = "http://abc/def/ghi.bam"
-
-    val downloader = AccessUrlDownloader(
-      accessUrl = AccessUrl(url = fakeAccessUrl, headers = None),
-      downloadLoc = fakeDownloadLocation,
-      hashes = None
-    )
-
-    val expected = s"""mkdir -p $$(dirname '$fakeDownloadLocation') && rm -f '$fakeDownloadLocation' && getm --checksum-algorithm 'null' --checksum null --filepath '$fakeDownloadLocation' '$fakeAccessUrl'""".validNel
-
-    downloader.generateDownloadScript shouldBe expected
-  }
-
-  {
-    val results = Table(
-      ("exitCode", "stderr", "download result"),
-      (0, "", DownloadSuccess),
-      // In `getm` version 0.0.4 checksum failures currently exit 0.
-      (0, "oh me oh my: AssertionError: Checksum failed!!!", ChecksumFailure),
-      // Unrecognized because of non-zero exit code without an HTTP status, despite what looks like a checksum failure.
-      (1, "oh me oh my: AssertionError: Checksum failed!!!", UnrecognizedRetryableDownloadFailure(ExitCode(1))),
-      // Unrecognized because of zero exit status with stderr that does not look like a checksum failure.
-      (0, "what the", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
-      // Unrecognized because of non-zero exit code without an HTTP status.
-      (1, " foobar ", UnrecognizedRetryableDownloadFailure(ExitCode(1))),
-      // Unrecognized because of zero exit status with stderr that does not look like a checksum failure.
-      (0, """ERROR:getm.cli possibly some words "status_code": 503 words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
-      // Recognized because of non-zero exit status and an HTTP status.
-      (1, """ERROR:getm.cli possibly some words "status_code": 503 words""", RecognizedRetryableDownloadFailure(ExitCode(1))),
-      // Recognized because of non-zero exit status and an HTTP status.
-      (1, """ERROR:getm.cli possibly some words "status_code": 408 more words""", RecognizedRetryableDownloadFailure(ExitCode(1))),
-      // Recognized and non-retryable because of non-zero exit status and 404 HTTP status.
-      (1, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", FatalDownloadFailure(ExitCode(1))),
-      // Unrecognized because of zero exit status and 404 HTTP status.
-      (0, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
-    )
-
-    val accessUrlDownloader = AccessUrlDownloader(null, null, null)
-
-    forAll(results) { (exitCode, stderr, expected) =>
-      it should s"produce $expected for exitCode $exitCode and stderr '$stderr'" in {
-        accessUrlDownloader.toDownloadResult(GetmResult(exitCode, stderr)) shouldBe expected
-      }
-    }
-  }
-}
-*/
\ No newline at end of file
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
index 491aa88cc0b..7b96ece8d0a 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala
@@ -1,11 +1,14 @@
 package drs.localizer.downloaders
 
-import cats.effect.IO
+import cats.effect.{ExitCode, IO}
 import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
+import org.scalatest.prop.TableDrivenPropertyChecks._
 import drs.localizer.{ResolvedDrsUrl, URIType}
+
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
+
 import java.nio.file.Path
 
 class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
@@ -76,4 +79,36 @@ class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec w
     val expected = s"""getm --manifest ${filepath.toString}"""
     downloader.generateGetmCommand(filepath) shouldBe expected
   }
+
+  {
+    val results = Table(
+      ("exitCode", "stderr", "download result"),
+      (0, "", DownloadSuccess),
+      // In `getm` version 0.0.4 checksum failures currently exit 0.
+      (0, "oh me oh my: AssertionError: Checksum failed!!!", ChecksumFailure),
+      // Unrecognized because of non-zero exit code without an HTTP status, despite what looks like a checksum failure.
+      (1, "oh me oh my: AssertionError: Checksum failed!!!", UnrecognizedRetryableDownloadFailure(ExitCode(1))),
+      // Unrecognized because of zero exit status with stderr that does not look like a checksum failure.
+      (0, "what the", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
+      // Unrecognized because of non-zero exit code without an HTTP status.
+      (1, " foobar ", UnrecognizedRetryableDownloadFailure(ExitCode(1))),
+      // Unrecognized because of zero exit status with stderr that does not look like a checksum failure.
+      (0, """ERROR:getm.cli possibly some words "status_code": 503 words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
+      // Recognized because of non-zero exit status and an HTTP status.
+      (1, """ERROR:getm.cli possibly some words "status_code": 503 words""", RecognizedRetryableDownloadFailure(ExitCode(1))),
+      // Recognized because of non-zero exit status and an HTTP status.
+      (1, """ERROR:getm.cli possibly some words "status_code": 408 more words""", RecognizedRetryableDownloadFailure(ExitCode(1))),
+      // Recognized and non-retryable because of non-zero exit status and 404 HTTP status.
+      (1, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", FatalDownloadFailure(ExitCode(1))),
+      // Unrecognized because of zero exit status and 404 HTTP status.
+      (0, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))),
+    )
+    val bulkDownloader = BulkAccessUrlDownloader(null)
+
+    forAll(results) { (exitCode, stderr, expected) =>
+      it should s"produce $expected for exitCode $exitCode and stderr '$stderr'" in {
+        bulkDownloader.toDownloadResult(GetmResult(exitCode, stderr)) shouldBe expected
+      }
+    }
+  }
 }

From 5e876d994c0cc93528f0db43502d9db58027de32 Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 15 Sep 2023 12:47:34 -0400
Subject: [PATCH 35/36] version for bee testing

---
 .../drs/localizer/DrsLocalizerMain.scala      |   9 +-
 .../drs/localizer/DrsLocalizerMainSpec.scala  | 214 ------------------
 2 files changed, 6 insertions(+), 217 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
index 6148c53dafd..3d99538f614 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala
@@ -8,9 +8,10 @@ import cloud.nio.impl.drs._
 import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff}
 import com.typesafe.scalalogging.StrictLogging
 import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google}
-import drs.localizer.DrsLocalizerMain.toValidatedUriType
+import drs.localizer.DrsLocalizerMain.{defaultNumRetries, toValidatedUriType}
 import drs.localizer.downloaders._
 import org.apache.commons.csv.{CSVFormat, CSVParser}
+
 import java.io.File
 import java.nio.charset.Charset
 import scala.concurrent.duration._
@@ -40,8 +41,10 @@ object DrsLocalizerMain extends IOApp with StrictLogging {
 
   def buildParser(): scopt.OptionParser[CommandLineArguments] = new CommandLineParser()
 
+  // Default retry parameters for resolving a DRS url
+  val defaultNumRetries: Int = 5
   val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff(
-    initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
+    initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2)
 
   val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory {
     override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader =
@@ -191,7 +194,7 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]],
     unresolvedUrls.flatMap { unresolvedList =>
       getDrsPathResolver.flatMap { resolver =>
         unresolvedList.map { unresolvedUrl =>
-          resolveWithRetries(resolver, unresolvedUrl, 5, Option(defaultBackoff))
+          resolveWithRetries(resolver, unresolvedUrl, defaultNumRetries, Option(defaultBackoff))
         }.traverse(identity)
       }
     }
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 07db22423a8..47bd4b1790d 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -239,220 +239,6 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       DrsLocalizerMain.toValidatedUriType(emptyResponse.accessUrl, emptyResponse.gsUri)
     } should have message "DRS response did not contain any URLs"
   }
-
-/*
-    it should "not retry on access URL download success" in {
-      var actualAttempts = 0
-
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          super.resolveAndDownload(downloaderFactory)
-        }
-      }
-      val accessUrlDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(DownloadSuccess)
-      })
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          // This test path should never ask for the GCS downloader
-          throw new RuntimeException("test failure")
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      drsLocalizer.resolveAndDownloadWithRetries(
-        downloadRetries = 3,
-        checksumRetries = 1,
-        downloaderFactory = downloaderFactory,
-        backoff = None
-      ).unsafeRunSync() shouldBe DownloadSuccess
-
-      actualAttempts shouldBe 1
-    }
-
-    it should "retry an appropriate number of times for regular retryable access URL download failures" in {
-      var actualAttempts = 0
-
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          super.resolveAndDownload(downloaderFactory)
-        }
-      }
-      val accessUrlDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
-      })
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          // This test path should never ask for the GCS downloader
-          throw new RuntimeException("test failure")
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      assertThrows[Throwable] {
-        drsLocalizer.resolveAndDownloadWithRetries(
-          downloadRetries = 3,
-          checksumRetries = 1,
-          downloaderFactory = downloaderFactory,
-          backoff = None
-        ).unsafeRunSync()
-      }
-
-      actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
-    }
-
-    it should "retry an appropriate number of times for fatal retryable access URL download failures" in {
-      var actualAttempts = 0
-
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          IO.raiseError(new RuntimeException("testing: fatal error") with FatalRetryDisposition)
-        }
-      }
-
-      val accessUrlDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0)))
-      })
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          // This test path should never ask for the GCS downloader
-          throw new RuntimeException("test failure")
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      assertThrows[Throwable] {
-        drsLocalizer.resolveAndDownloadWithRetries(
-          downloadRetries = 3,
-          checksumRetries = 1,
-          downloaderFactory = downloaderFactory,
-          backoff = None
-        ).unsafeRunSync()
-      }
-
-      actualAttempts shouldBe 1 // 1 and done with a fatal exception
-    }
-
-    it should "not retry on GCS URI download success" in {
-      var actualAttempts = 0
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          super.resolveAndDownload(downloaderFactory)
-        }
-      }
-      val gcsUriDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(DownloadSuccess)
-      })
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          gcsUriDownloader
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      drsLocalizer.resolveAndDownloadWithRetries(
-        downloadRetries = 3,
-        checksumRetries = 1,
-        downloaderFactory = downloaderFactory,
-        backoff = None).unsafeRunSync()
-
-      actualAttempts shouldBe 1
-    }
-
-    it should "retry an appropriate number of times for retryable GCS URI download failures" in {
-      var actualAttempts = 0
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          super.resolveAndDownload(downloaderFactory)
-        }
-      }
-      val gcsUriDownloader = IO.pure(new Downloader {
-        override def download: IO[DownloadResult] =
-          IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(1)))
-      })
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          gcsUriDownloader
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      assertThrows[Throwable] {
-        drsLocalizer.resolveAndDownloadWithRetries(
-          downloadRetries = 3,
-          checksumRetries = 1,
-          downloaderFactory = downloaderFactory,
-          backoff = None).unsafeRunSync()
-      }
-
-      actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts
-    }
-
-    it should "retry an appropriate number of times for checksum failures" in {
-      var actualAttempts = 0
-      val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) {
-        override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = {
-          actualAttempts = actualAttempts + 1
-          super.resolveAndDownload(downloaderFactory)
-        }
-      }
-
-      val downloaderFactory = new DownloaderFactory {
-        override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = {
-          // This test path should never ask for the GCS URI downloader.
-          throw new RuntimeException("test failure")
-        }
-
-        override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): IO[Downloader] = {
-          // This test path should never ask for the Bulk downloader
-          throw new RuntimeException("test failure")
-        }
-      }
-
-      assertThrows[Throwable] {
-        drsLocalizer.resolveAndDownloadWithRetries(
-          downloadRetries = 3,
-          checksumRetries = 1,
-          downloaderFactory = downloaderFactory,
-          backoff = None).unsafeRunSync()
-      }
-      actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts
-    }
-    */
 }
 
 object MockDrsPaths {

From c236c87ff7afeb3ee9982d5718f5bc2d19315a2b Mon Sep 17 00:00:00 2001
From: Tom Wiseman <twiseman@broadinstitute.org>
Date: Fri, 15 Sep 2023 13:25:57 -0400
Subject: [PATCH 36/36] includes, better retries

---
 .../scala/drs/localizer/downloaders/GcsUriDownloader.scala  | 6 ++++--
 .../src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala | 2 --
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
index 3d82c54595f..8991e79f5fd 100644
--- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
+++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala
@@ -14,10 +14,12 @@ case class GcsUriDownloader(gcsUrl: String,
                             downloadLoc: String,
                             requesterPaysProjectIdOption: Option[String]) extends Downloader with StrictLogging {
 
+  val defaultNumRetries: Int = 5
   val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff(
-    initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2)
+    initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2)
+
   override def download: IO[DownloadResult] = {
-    downloadWithRetries(5, Option(defaultBackoff))
+    downloadWithRetries(defaultNumRetries, Option(defaultBackoff))
   }
 
   def runDownloadCommand: IO[DownloadResult] = {
diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
index 47bd4b1790d..52fa4c99330 100644
--- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
+++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala
@@ -4,12 +4,10 @@ import cats.data.NonEmptyList
 import cats.effect.{ExitCode, IO}
 import cats.syntax.validated._
 import drs.localizer.MockDrsPaths.{fakeAccessUrls, fakeDrsUrlWithGcsResolutionOnly, fakeGoogleUrls}
-//import cloud.nio.impl.drs.DrsPathResolver.FatalRetryDisposition
 import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsResolverField, DrsResolverResponse}
 import common.assertion.CromwellTimeoutSpec
 import common.validation.ErrorOr.ErrorOr
 import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes}
-//import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes
 import drs.localizer.downloaders._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers