add json dependency

WX-1210 Added JIRA ID for Cromwhelm auto commit message (#7180)

make it scala

lots of scaffolding

WX-1179 DRS image build updates, remove self-hosted runners (#7179)

WX-1210-action-fix Use PR title to find JIRA ID for cromwhelm commit message (#7184)

WX-1110-query-fix Corrected Query to pull in attributes outside of executionStatus and backendStatus (#7185)

WX-1179 GCP Batch new base (#7177)

Co-authored-by: dspeck <dspeck@burwood.com>
Co-authored-by: jarroyo28 <johnarroyo45@gmail.com>
Co-authored-by: Jacob Jennings <53196131+jakejennings@users.noreply.github.com>
Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>
Co-authored-by: Tom Wiseman <twiseman@broadinstitute.org>
Co-authored-by: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Co-authored-by: Christian Freitas <cfreitas@broadinstitute.org>
Co-authored-by: Tristan Garwood <tristan.garwood@gmail.com>

WM-2100: Update terra-helmfile on publish (#7187)

WM-2100: Correct commit and PR description of terra-helmfile auto update (#7188)

forward progress

stash

more progress

json

WX-1230 Ignore `invalidate_bad_caches_use_good_local.test` (#7197)

WX-1078 ACR support (#7192)

WX-1179 Enable GCP Batch Integration Tests (#7199)

Co-authored-by: Adam Nichols <aednichols@gmail.com>
Co-authored-by: Adam Nichols <anichols@broadinstitute.org>

WX-1179 GCP Batch Docs Update (#7196)

Co-authored-by: Jacob Jennings <53196131+jakejennings@users.noreply.github.com>
Co-authored-by: Adam Nichols <aednichols@gmail.com>
Co-authored-by: Adam Nichols <anichols@broadinstitute.org>

stash

expired tokens

ID-734 Increase Timeout for DRSHub Communication (#7198)

[WX-1156] internal_path_prefix for TES 4.4 (#7190)

think about stuff

undo sins

undo one more sin

[WM-2184] Remove brackets from Jira ID (#7206)

WX-1153 Azure blob read md5 from metadata for large files (#7204)

Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>

WX-1156 Fix internal_path_prefix (#7208)

Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>

WX-1256 Temporarily turn off engine hashing for blob files (#7209)

Co-authored-by: Adam Nichols <aednichols@gmail.com>

WX-1173 Reopen filesystem for blob storage outside workspace (#7178)

Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>
Co-authored-by: Tom Wiseman <twiseman@broadinstitute.org>
Co-authored-by: Adam Nichols <aednichols@gmail.com>

WX-1174 Adjust NIO Copy functionality (#7207)

Co-authored-by: Adam Nichols <anichols@broadinstitute.org>

[WX-1168] TES Log Paths (#7210)

deck chairs

somethin that kinda works

working test

lots of good cleanup

time for tests

remove conf

working on tests

test progress

much cleaner

oops

bunch of tests

more better tests

.github/workflows/chart_update_on_merge.yml

@@ -9,8 +9,14 @@ jobs:
   chart-update:
     name: Cromwhelm Chart Auto Updater
     if: github.event.pull_request.merged == true
-    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    runs-on: ubuntu-latest
     steps:
+    - name: Fetch Jira ID from the commit message
+      id: fetch-jira-id
+      run: |
+        JIRA_ID=$(echo '${{ github.event.pull_request.title }}' | grep -Eo '[A-Z][A-Z]+-[0-9]+' | xargs echo -n | tr '[:space:]' ',')
+        [[ -z "$JIRA_ID" ]] && { echo "No Jira ID found in $1" ; exit 1; }
+        echo "JIRA_ID=$JIRA_ID" >> $GITHUB_OUTPUT
     - name: Clone Cromwell
       uses: actions/checkout@v2
       with:
@@ -69,7 +75,7 @@ jobs:
         repository: broadinstitute/terra-helmfile
         event-type: update-service
         client-payload: '{"service": "cromiam", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}'
-    - name: Edit & push chart
+    - name: Edit & push cromwhelm chart
       env:
         BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
       run: |
@@ -82,5 +88,48 @@ jobs:
         git diff
         git config --global user.name "broadbot"
         git config --global user.email "broadbot@broadinstitute.org"
-        git commit -am "Auto update to Cromwell $CROMWELL_VERSION"
+        git commit -am "${{ steps.fetch-jira-id.outputs.JIRA_ID }}: Auto update to Cromwell $CROMWELL_VERSION"
         git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/cromwhelm.git main
+        cd -
+
+    - name: Clone terra-helmfile
+      uses: actions/checkout@v3
+      with:
+        repository: broadinstitute/terra-helmfile
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+        path: terra-helmfile
+
+    - name: Update workflows-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/workflows-app/values.yaml
+        cd -
+
+    - name: Update cromwell-runner-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/cromwell-runner-app/values.yaml
+        cd -
+        
+
+    - name: Make PR in terra-helmfile
+      env:
+        BROADBOT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        GH_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+      run: |
+        set -e
+        JIRA_ID=${{ steps.fetch-jira-id.outputs.JIRA_ID }}
+        if [[ $JIRA_ID == "missing" ]]; then
+          echo "JIRA_ID missing, PR to terra-helmfile will not be created"
+          exit 0;
+        fi
+        cd terra-helmfile
+        git checkout -b ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        git config --global user.name "broadbot"
+        git config --global user.email "broadbot@broadinstitute.org"
+        git commit -am "${JIRA_ID}: Auto update Cromwell to $CROMWELL_VERSION in workflows-app and cromwell-runner-app"
+        git push -u origin ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        gh pr create --title "${JIRA_ID}: auto update Cromwell version to $CROMWELL_VERSION in workflows-app and cromwell-runner-app" --body "${JIRA_ID} helm chart update" --label "automerge"
+        cd -

.github/workflows/cromwell_unit_tests.yml

@@ -28,6 +28,10 @@ jobs:
 
     #Invoke SBT to run all unit tests for Cromwell. 
     - name: Run tests
+      env:
+        AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }}
+        AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }}
+        AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }}
       run: |
         set -e
         sbt "test"

.github/workflows/docker_build_test.yml

@@ -17,7 +17,7 @@ permissions:
 jobs:
   sbt-build:
     name: sbt docker build
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Clone Cromwell
         uses: actions/checkout@v2

.github/workflows/integration_tests.yml

@@ -24,7 +24,11 @@ jobs:
       #Each will be launched on its own runner so they can occur in parallel.
       #Friendly names are displayed on the Github UI and aren't used anywhere else.
       matrix:
+        # Batch test fixes to land later
         include:
+          - build_type: centaurGcpBatch
+            build_mysql: 5.7
+            friendly_name: Centaur GCP Batch with MySQL 5.7
           - build_type: centaurPapiV2beta
             build_mysql: 5.7
             friendly_name: Centaur Papi V2 Beta with MySQL 5.7

.github/workflows/make_publish_prs.yml

@@ -16,7 +16,7 @@ on:
 jobs:
   make-firecloud-develop-pr:
     name: Create firecloud-develop PR
-    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    runs-on: ubuntu-latest
     steps:
       - name: Clone firecloud-develop
         uses: actions/checkout@v2
@@ -70,4 +70,3 @@ jobs:
                 'It updates cromwell from version ${{ github.event.inputs.old_cromwell_version }} to ${{ github.event.inputs.new_cromwell_version }}.'
               ].join('\n')
             });
-

azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java

@@ -3,12 +3,6 @@
 
 package com.azure.storage.blob.nio;
 
-import com.azure.core.util.logging.ClientLogger;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.BlobListDetails;
-import com.azure.storage.blob.models.ListBlobsOptions;
-
 import java.io.IOException;
 import java.nio.file.DirectoryIteratorException;
 import java.nio.file.DirectoryStream;
@@ -18,6 +12,12 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobListDetails;
+import com.azure.storage.blob.models.ListBlobsOptions;
+
 /**
  * A type for iterating over the contents of a directory.
  *
@@ -88,7 +88,7 @@ private static class AzureDirectoryIterator implements Iterator<Path> {
             if (path.isRoot()) {
                 String containerName = path.toString().substring(0, path.toString().length() - 1);
                 AzureFileSystem afs = ((AzureFileSystem) path.getFileSystem());
-                containerClient = ((AzureFileStore) afs.getFileStore(containerName)).getContainerClient();
+                containerClient = ((AzureFileStore) afs.getFileStore()).getContainerClient();
             } else {
                 AzureResource azureResource = new AzureResource(path);
                 listOptions.setPrefix(azureResource.getBlobClient().getBlobName() + AzureFileSystem.PATH_SEPARATOR);

azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java

@@ -3,19 +3,6 @@
 
 package com.azure.storage.blob.nio;
 
-import com.azure.core.credential.AzureSasCredential;
-import com.azure.core.http.HttpClient;
-import com.azure.core.http.policy.HttpLogDetailLevel;
-import com.azure.core.http.policy.HttpPipelinePolicy;
-import com.azure.core.util.CoreUtils;
-import com.azure.core.util.logging.ClientLogger;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.implementation.util.BlobUserAgentModificationPolicy;
-import com.azure.storage.common.StorageSharedKeyCredential;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.azure.storage.common.policy.RetryPolicyType;
-
 import java.io.IOException;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
@@ -27,14 +14,31 @@
 import java.nio.file.attribute.FileAttributeView;
 import java.nio.file.attribute.UserPrincipalLookupService;
 import java.nio.file.spi.FileSystemProvider;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
+
+import com.azure.core.credential.AzureSasCredential;
+import com.azure.core.http.HttpClient;
+import com.azure.core.http.policy.HttpLogDetailLevel;
+import com.azure.core.http.policy.HttpPipelinePolicy;
+import com.azure.core.util.CoreUtils;
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.implementation.util.BlobUserAgentModificationPolicy;
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
 
 /**
  * Implement's Java's {@link FileSystem} interface for Azure Blob Storage.
@@ -67,6 +71,11 @@ public final class AzureFileSystem extends FileSystem {
      */
     public static final String AZURE_STORAGE_SAS_TOKEN_CREDENTIAL = "AzureStorageSasTokenCredential";
 
+    /**
+     * Expected type: String
+     */
+    public static final String AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL = "AzureStoragePublicAccessCredential";
+
     /**
      * Expected type: com.azure.core.http.policy.HttpLogLevelDetail
      */
@@ -159,10 +168,12 @@ public final class AzureFileSystem extends FileSystem {
     private final Long putBlobThreshold;
     private final Integer maxConcurrencyPerRequest;
     private final Integer downloadResumeRetries;
-    private final Map<String, FileStore> fileStores;
     private FileStore defaultFileStore;
     private boolean closed;
 
+    private AzureSasCredential currentActiveSasCredential;
+    private Instant expiry;
+
     AzureFileSystem(AzureFileSystemProvider parentFileSystemProvider, String endpoint, Map<String, ?> config)
             throws IOException {
         // A FileSystem should only ever be instantiated by a provider.
@@ -179,9 +190,10 @@ public final class AzureFileSystem extends FileSystem {
             this.putBlobThreshold = (Long) config.get(AZURE_STORAGE_PUT_BLOB_THRESHOLD);
             this.maxConcurrencyPerRequest = (Integer) config.get(AZURE_STORAGE_MAX_CONCURRENCY_PER_REQUEST);
             this.downloadResumeRetries = (Integer) config.get(AZURE_STORAGE_DOWNLOAD_RESUME_RETRIES);
+            this.currentActiveSasCredential = (AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL);
 
             // Initialize and ensure access to FileStores.
-            this.fileStores = this.initializeFileStores(config);
+            this.defaultFileStore = this.initializeFileStore(config);
         } catch (RuntimeException e) {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("There was an error parsing the "
                 + "configurations map. Please ensure all fields are set to a legal value of the correct type.", e));
@@ -221,7 +233,7 @@ public FileSystemProvider provider() {
     @Override
     public void close() throws IOException {
         this.closed = true;
-        this.parentFileSystemProvider.closeFileSystem(this.getFileSystemUrl());
+        this.parentFileSystemProvider.closeFileSystem(this.getFileSystemUrl() + "/" + defaultFileStore.name());
     }
 
     /**
@@ -282,9 +294,7 @@ public Iterable<Path> getRootDirectories() {
         If the file system was set to use all containers in the account, the account will be re-queried and the
         list may grow or shrink if containers were added or deleted.
          */
-        return fileStores.keySet().stream()
-            .map(name -> this.getPath(name + AzurePath.ROOT_DIR_SUFFIX))
-            .collect(Collectors.toList());
+        return Arrays.asList(this.getPath(defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX));
     }
 
     /**
@@ -304,7 +314,7 @@ public Iterable<FileStore> getFileStores() {
         If the file system was set to use all containers in the account, the account will be re-queried and the
         list may grow or shrink if containers were added or deleted.
          */
-        return this.fileStores.values();
+        return Arrays.asList(defaultFileStore);
     }
 
     /**
@@ -397,6 +407,12 @@ private BlobServiceClient buildBlobServiceClient(String endpoint, Map<String, ?>
             builder.credential((StorageSharedKeyCredential) config.get(AZURE_STORAGE_SHARED_KEY_CREDENTIAL));
         } else if (config.containsKey(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL)) {
             builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL));
+            this.setExpiryFromSAS((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL));
+        } else if (config.containsKey(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL)) {
+            // The Blob Service Client Builder requires at least one kind of authentication to make requests
+            // For public files however, this is unnecessary. This key-value pair is to denote the case
+            // explicitly when we supply a placeholder SAS credential to bypass this requirement.
+            builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL));
         } else {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException(String.format("No credentials were "
                     + "provided. Please specify one of the following when constructing an AzureFileSystem: %s, %s.",
@@ -430,23 +446,17 @@ private BlobServiceClient buildBlobServiceClient(String endpoint, Map<String, ?>
         return builder.buildClient();
     }
 
-    private Map<String, FileStore> initializeFileStores(Map<String, ?> config) throws IOException {
-        String fileStoreNames = (String) config.get(AZURE_STORAGE_FILE_STORES);
-        if (CoreUtils.isNullOrEmpty(fileStoreNames)) {
+    private FileStore initializeFileStore(Map<String, ?> config) throws IOException {
+        String fileStoreName = (String) config.get(AZURE_STORAGE_FILE_STORES);
+        if (CoreUtils.isNullOrEmpty(fileStoreName)) {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("The list of FileStores cannot be "
                 + "null."));
         }
 
         Boolean skipConnectionCheck = (Boolean) config.get(AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK);
         Map<String, FileStore> fileStores = new HashMap<>();
-        for (String fileStoreName : fileStoreNames.split(",")) {
-            FileStore fs = new AzureFileStore(this, fileStoreName, skipConnectionCheck);
-            if (this.defaultFileStore == null) {
-                this.defaultFileStore = fs;
-            }
-            fileStores.put(fileStoreName, fs);
-        }
-        return fileStores;
+        this.defaultFileStore = new AzureFileStore(this, fileStoreName, skipConnectionCheck);
+        return this.defaultFileStore;
     }
 
     @Override
@@ -470,12 +480,11 @@ Path getDefaultDirectory() {
         return this.getPath(this.defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX);
     }
 
-    FileStore getFileStore(String name) throws IOException {
-        FileStore store = this.fileStores.get(name);
-        if (store == null) {
-            throw LoggingUtility.logError(LOGGER, new IOException("Invalid file store: " + name));
+    FileStore getFileStore() throws IOException {
+        if (this.defaultFileStore == null) {
+            throw LoggingUtility.logError(LOGGER, new IOException("FileStore not initialized"));
         }
-        return store;
+        return defaultFileStore;
     }
 
     Long getBlockSize() {
@@ -489,4 +498,32 @@ Long getPutBlobThreshold() {
     Integer getMaxConcurrencyPerRequest() {
         return this.maxConcurrencyPerRequest;
     }
+
+    public String createSASAppendedURL(String url) throws IllegalStateException {
+        if (Objects.isNull(currentActiveSasCredential)) {
+            throw new IllegalStateException("No current active SAS credential present");
+        }
+        return url + "?" + currentActiveSasCredential.getSignature();
+    }
+
+    public Optional<Instant> getExpiry() {
+        return Optional.ofNullable(expiry);
+    }
+
+    private void setExpiryFromSAS(AzureSasCredential token) {
+        List<String> strings = Arrays.asList(token.getSignature().split("&"));
+        Optional<String> expiryString = strings.stream()
+            .filter(s -> s.startsWith("se"))
+            .findFirst()
+            .map(s -> s.replaceFirst("se=",""))
+            .map(s -> s.replace("%3A", ":"));
+        this.expiry = expiryString.map(es -> Instant.parse(es)).orElse(null);
+    }
+
+    public boolean isExpired(Duration buffer) {
+        return Optional.ofNullable(this.expiry)
+            .map(e -> Instant.now().plus(buffer).isAfter(e))
+            .orElse(true);
+
+    }
 }