diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index d92ee8ef102..2788681fd08 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -11,6 +11,7 @@ jobs: if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: + - uses: sbt/setup-sbt@v1 - name: Clone Cromwell uses: actions/checkout@v4 with: diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 88951871d8f..776685f7e98 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -22,6 +22,7 @@ jobs: steps: - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: sbt/setup-sbt@v1 - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index d3e5369fca5..2ea9d9a4970 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -25,6 +25,7 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} path: cromwell + - uses: sbt/setup-sbt@v1 - uses: actions/setup-java@v4 with: distribution: 'temurin' diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index bf13316158a..d898fa222ef 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -93,9 +93,9 @@ jobs: friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 - build_type: referenceDiskManifestBuilderApp friendly_name: Reference Disk Manifest Builder App - - build_type: centaurSlurm - build_mysql: 5.7 - friendly_name: "Centaur Slurm with MySQL 5.7" +# - build_type: centaurSlurm +# build_mysql: 5.7 +# friendly_name: "Centaur Slurm with MySQL 5.7" - build_type: centaurBlob build_mysql: 5.7 friendly_name: Centaur Blob @@ -114,6 +114,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: + - uses: sbt/setup-sbt@v1 - uses: actions/checkout@v3 # checkout the cromwell repo with: ref: ${{ inputs.target-branch }} diff --git a/.github/workflows/scalafmt-check.yml b/.github/workflows/scalafmt-check.yml index 3730d2ffc8f..e547fcc85a4 100644 --- a/.github/workflows/scalafmt-check.yml +++ b/.github/workflows/scalafmt-check.yml @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: + - uses: sbt/setup-sbt@v1 - uses: actions/checkout@v3 with: ref: ${{ inputs.target-branch }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1704b3b826c..0e190503d22 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -17,6 +17,7 @@ jobs: steps: - uses: actions/checkout@v2 + - uses: sbt/setup-sbt@v1 # fetch SBT package - uses: actions/setup-java@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 36abc519ed7..3f5fa58aaa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,9 +55,17 @@ The `IX_WORKFLOW_STORE_ENTRY_WS` index is removed from `WORKFLOW_STORE_ENTRY`. The index had low cardinality and workflow pickup is faster without it. Migration time depends on workflow store size, but should be very fast for most installations. Terminal workflows are removed from the workflow store, so only running workflows contribute to the cost. -### Bug fixes and small changes +#### Index additions - * Changed default boot disk size from 10GB to 20GB in PipelinesAPI and Google Batch backends +The `IX_METADATA_ENTRY_WEU_MK` index is added to `METADATA_ENTRY`. In pre-release testing, the migration proceeded at about 3 million rows per minute. Please plan downtime accordingly. + +#### Unique constraint addition + +The `UC_GROUP_METRICS_ENTRY_GI` constraint has been added to column `GROUP_ID` in `GROUP_METRICS_ENTRY` table. This additionally enforces at database level that the `GROUP_ID` column always contains unique values. + +### Reduce errors from boot disk filling up on Google Lifesciences API + + * If Cromwell can't determine the size of the user command Docker image, it will increase Lifesciences API boot disk size by 30GB rather than 0. This should reduce incidence of tasks failing due to boot disk filling up. #### Improved `size()` function performance on arrays diff --git a/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test b/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test index f7d59e4b390..dc3d297f6fc 100644 --- a/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test +++ b/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test @@ -11,8 +11,8 @@ files { metadata { status: Succeeded - "outputs.docker_size_dockerhub.large_dockerhub_image_with_hash.bootDiskSize": 27 - "outputs.docker_size_dockerhub.large_dockerhub_image_with_tag.bootDiskSize": 27 + "outputs.docker_size_dockerhub.large_dockerhub_image_with_hash.bootDiskSize": 17 + "outputs.docker_size_dockerhub.large_dockerhub_image_with_tag.bootDiskSize": 17 } workflowType: WDL diff --git a/centaur/src/main/resources/standardTestCases/docker_size_gcr.test b/centaur/src/main/resources/standardTestCases/docker_size_gcr.test index 2e8c2e1b2d3..7399089f307 100644 --- a/centaur/src/main/resources/standardTestCases/docker_size_gcr.test +++ b/centaur/src/main/resources/standardTestCases/docker_size_gcr.test @@ -11,8 +11,8 @@ files { metadata { status: Succeeded - "outputs.docker_size_gcr.large_gcr_image_with_hash.bootDiskSize": 27 - "outputs.docker_size_gcr.large_gcr_image_with_tag.bootDiskSize": 27 + "outputs.docker_size_gcr.large_gcr_image_with_hash.bootDiskSize": 17 + "outputs.docker_size_gcr.large_gcr_image_with_tag.bootDiskSize": 17 } workflowType: WDL diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala new file mode 100644 index 00000000000..6a3d4acaa62 --- /dev/null +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -0,0 +1,28 @@ +package cromwell.cloudsupport.azure + +import com.azure.core.management.AzureEnvironment +import com.typesafe.config.ConfigFactory +import net.ceedubs.ficus.Ficus._ + +object AzureConfiguration { + private val conf = ConfigFactory.load().getConfig("azure") + val azureEnvironment = + AzureEnvironmentConverter.fromString( + conf.as[Option[String]]("azure-environment").getOrElse(AzureEnvironmentConverter.Azure) + ) + val azureTokenScopeManagement = conf.as[String]("token-scope-management") +} + +object AzureEnvironmentConverter { + val Azure: String = "AzureCloud" + val AzureGov: String = "AzureUSGovernmentCloud" + val AzureChina: String = "AzureChinaCloud" + + def fromString(s: String): AzureEnvironment = s match { + case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT + case AzureChina => AzureEnvironment.AZURE_CHINA + // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default + case Azure => AzureEnvironment.AZURE + case _ => AzureEnvironment.AZURE + } +} diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala index d3d66e1bafc..034e464aa54 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala @@ -2,7 +2,6 @@ package cromwell.cloudsupport.azure import cats.implicits.catsSyntaxValidatedId import com.azure.core.credential.TokenRequestContext -import com.azure.core.management.AzureEnvironment import com.azure.core.management.profile.AzureProfile import com.azure.identity.DefaultAzureCredentialBuilder import common.validation.ErrorOr.ErrorOr @@ -20,8 +19,8 @@ case object AzureCredentials { final val tokenAcquisitionTimeout = 5.seconds - val azureProfile = new AzureProfile(AzureEnvironment.AZURE) - val tokenScope = "https://management.azure.com/.default" + val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment) + val tokenScope = AzureConfiguration.azureTokenScopeManagement private def tokenRequestContext: TokenRequestContext = { val trc = new TokenRequestContext() diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala index dd379ed3564..1febd5d77c0 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala @@ -1,6 +1,5 @@ package cromwell.cloudsupport.azure -import com.azure.core.management.AzureEnvironment import com.azure.core.management.profile.AzureProfile import com.azure.identity.DefaultAzureCredentialBuilder import com.azure.resourcemanager.AzureResourceManager @@ -33,7 +32,7 @@ object AzureUtils { .map(Success(_)) .getOrElse(Failure(new Exception("Could not parse storage account"))) - val azureProfile = new AzureProfile(AzureEnvironment.AZURE) + val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment) def azureCredentialBuilder = new DefaultAzureCredentialBuilder() .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 9da59f16b7b..057b20a9642 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -330,6 +330,11 @@ call-caching { max-failed-copy-attempts = 1000000 } +azure { + azure-environment = "AzureCloud" + token-scope-management = "https://management.azure.com/.default" +} + google { application-name = "cromwell" diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 24c1943eec7..81036465176 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -92,6 +92,7 @@ + + + + + + + + + + + + diff --git a/database/migration/src/main/resources/sql_metadata_changelog.xml b/database/migration/src/main/resources/sql_metadata_changelog.xml index 0989ec2199a..11437a2ce3a 100644 --- a/database/migration/src/main/resources/sql_metadata_changelog.xml +++ b/database/migration/src/main/resources/sql_metadata_changelog.xml @@ -19,6 +19,7 @@ +