diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml
index d92ee8ef102..2788681fd08 100644
--- a/.github/workflows/chart_update_on_merge.yml
+++ b/.github/workflows/chart_update_on_merge.yml
@@ -11,6 +11,7 @@ jobs:
if: github.event.pull_request.merged == true
runs-on: ubuntu-latest
steps:
+ - uses: sbt/setup-sbt@v1
- name: Clone Cromwell
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml
index 88951871d8f..776685f7e98 100644
--- a/.github/workflows/cromwell_unit_tests.yml
+++ b/.github/workflows/cromwell_unit_tests.yml
@@ -22,6 +22,7 @@ jobs:
steps:
- uses: actions/checkout@v3 # checkout the cromwell repo
+ - uses: sbt/setup-sbt@v1
- uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell.
with:
cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml
index d3e5369fca5..2ea9d9a4970 100644
--- a/.github/workflows/docker_build_test.yml
+++ b/.github/workflows/docker_build_test.yml
@@ -25,6 +25,7 @@ jobs:
repository: broadinstitute/cromwell
token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
path: cromwell
+ - uses: sbt/setup-sbt@v1
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index bf13316158a..d898fa222ef 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -93,9 +93,9 @@ jobs:
friendly_name: Centaur Engine Upgrade Local with MySQL 5.7
- build_type: referenceDiskManifestBuilderApp
friendly_name: Reference Disk Manifest Builder App
- - build_type: centaurSlurm
- build_mysql: 5.7
- friendly_name: "Centaur Slurm with MySQL 5.7"
+# - build_type: centaurSlurm
+# build_mysql: 5.7
+# friendly_name: "Centaur Slurm with MySQL 5.7"
- build_type: centaurBlob
build_mysql: 5.7
friendly_name: Centaur Blob
@@ -114,6 +114,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
+ - uses: sbt/setup-sbt@v1
- uses: actions/checkout@v3 # checkout the cromwell repo
with:
ref: ${{ inputs.target-branch }}
diff --git a/.github/workflows/scalafmt-check.yml b/.github/workflows/scalafmt-check.yml
index 3730d2ffc8f..e547fcc85a4 100644
--- a/.github/workflows/scalafmt-check.yml
+++ b/.github/workflows/scalafmt-check.yml
@@ -19,6 +19,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
+ - uses: sbt/setup-sbt@v1
- uses: actions/checkout@v3
with:
ref: ${{ inputs.target-branch }}
diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
index 1704b3b826c..0e190503d22 100644
--- a/.github/workflows/trivy.yml
+++ b/.github/workflows/trivy.yml
@@ -17,6 +17,7 @@ jobs:
steps:
- uses: actions/checkout@v2
+ - uses: sbt/setup-sbt@v1
# fetch SBT package
- uses: actions/setup-java@v4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36abc519ed7..3f5fa58aaa8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -55,9 +55,17 @@ The `IX_WORKFLOW_STORE_ENTRY_WS` index is removed from `WORKFLOW_STORE_ENTRY`.
The index had low cardinality and workflow pickup is faster without it. Migration time depends on workflow store size, but should be very fast for most installations. Terminal workflows are removed from the workflow store, so only running workflows contribute to the cost.
-### Bug fixes and small changes
+#### Index additions
- * Changed default boot disk size from 10GB to 20GB in PipelinesAPI and Google Batch backends
+The `IX_METADATA_ENTRY_WEU_MK` index is added to `METADATA_ENTRY`. In pre-release testing, the migration proceeded at about 3 million rows per minute. Please plan downtime accordingly.
+
+#### Unique constraint addition
+
+The `UC_GROUP_METRICS_ENTRY_GI` constraint has been added to column `GROUP_ID` in `GROUP_METRICS_ENTRY` table. This additionally enforces at database level that the `GROUP_ID` column always contains unique values.
+
+### Reduce errors from boot disk filling up on Google Lifesciences API
+
+ * If Cromwell can't determine the size of the user command Docker image, it will increase Lifesciences API boot disk size by 30GB rather than 0. This should reduce incidence of tasks failing due to boot disk filling up.
#### Improved `size()` function performance on arrays
diff --git a/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test b/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test
index f7d59e4b390..dc3d297f6fc 100644
--- a/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test
+++ b/centaur/src/main/resources/standardTestCases/docker_size_dockerhub.test
@@ -11,8 +11,8 @@ files {
metadata {
status: Succeeded
- "outputs.docker_size_dockerhub.large_dockerhub_image_with_hash.bootDiskSize": 27
- "outputs.docker_size_dockerhub.large_dockerhub_image_with_tag.bootDiskSize": 27
+ "outputs.docker_size_dockerhub.large_dockerhub_image_with_hash.bootDiskSize": 17
+ "outputs.docker_size_dockerhub.large_dockerhub_image_with_tag.bootDiskSize": 17
}
workflowType: WDL
diff --git a/centaur/src/main/resources/standardTestCases/docker_size_gcr.test b/centaur/src/main/resources/standardTestCases/docker_size_gcr.test
index 2e8c2e1b2d3..7399089f307 100644
--- a/centaur/src/main/resources/standardTestCases/docker_size_gcr.test
+++ b/centaur/src/main/resources/standardTestCases/docker_size_gcr.test
@@ -11,8 +11,8 @@ files {
metadata {
status: Succeeded
- "outputs.docker_size_gcr.large_gcr_image_with_hash.bootDiskSize": 27
- "outputs.docker_size_gcr.large_gcr_image_with_tag.bootDiskSize": 27
+ "outputs.docker_size_gcr.large_gcr_image_with_hash.bootDiskSize": 17
+ "outputs.docker_size_gcr.large_gcr_image_with_tag.bootDiskSize": 17
}
workflowType: WDL
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala
new file mode 100644
index 00000000000..6a3d4acaa62
--- /dev/null
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala
@@ -0,0 +1,28 @@
+package cromwell.cloudsupport.azure
+
+import com.azure.core.management.AzureEnvironment
+import com.typesafe.config.ConfigFactory
+import net.ceedubs.ficus.Ficus._
+
+object AzureConfiguration {
+ private val conf = ConfigFactory.load().getConfig("azure")
+ val azureEnvironment =
+ AzureEnvironmentConverter.fromString(
+ conf.as[Option[String]]("azure-environment").getOrElse(AzureEnvironmentConverter.Azure)
+ )
+ val azureTokenScopeManagement = conf.as[String]("token-scope-management")
+}
+
+object AzureEnvironmentConverter {
+ val Azure: String = "AzureCloud"
+ val AzureGov: String = "AzureUSGovernmentCloud"
+ val AzureChina: String = "AzureChinaCloud"
+
+ def fromString(s: String): AzureEnvironment = s match {
+ case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT
+ case AzureChina => AzureEnvironment.AZURE_CHINA
+ // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
+ case Azure => AzureEnvironment.AZURE
+ case _ => AzureEnvironment.AZURE
+ }
+}
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala
index d3d66e1bafc..034e464aa54 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala
@@ -2,7 +2,6 @@ package cromwell.cloudsupport.azure
import cats.implicits.catsSyntaxValidatedId
import com.azure.core.credential.TokenRequestContext
-import com.azure.core.management.AzureEnvironment
import com.azure.core.management.profile.AzureProfile
import com.azure.identity.DefaultAzureCredentialBuilder
import common.validation.ErrorOr.ErrorOr
@@ -20,8 +19,8 @@ case object AzureCredentials {
final val tokenAcquisitionTimeout = 5.seconds
- val azureProfile = new AzureProfile(AzureEnvironment.AZURE)
- val tokenScope = "https://management.azure.com/.default"
+ val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment)
+ val tokenScope = AzureConfiguration.azureTokenScopeManagement
private def tokenRequestContext: TokenRequestContext = {
val trc = new TokenRequestContext()
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala
index dd379ed3564..1febd5d77c0 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala
@@ -1,6 +1,5 @@
package cromwell.cloudsupport.azure
-import com.azure.core.management.AzureEnvironment
import com.azure.core.management.profile.AzureProfile
import com.azure.identity.DefaultAzureCredentialBuilder
import com.azure.resourcemanager.AzureResourceManager
@@ -33,7 +32,7 @@ object AzureUtils {
.map(Success(_))
.getOrElse(Failure(new Exception("Could not parse storage account")))
- val azureProfile = new AzureProfile(AzureEnvironment.AZURE)
+ val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment)
def azureCredentialBuilder = new DefaultAzureCredentialBuilder()
.authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint)
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 9da59f16b7b..057b20a9642 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -330,6 +330,11 @@ call-caching {
max-failed-copy-attempts = 1000000
}
+azure {
+ azure-environment = "AzureCloud"
+ token-scope-management = "https://management.azure.com/.default"
+}
+
google {
application-name = "cromwell"
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 24c1943eec7..81036465176 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -92,6 +92,7 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/database/migration/src/main/resources/sql_metadata_changelog.xml b/database/migration/src/main/resources/sql_metadata_changelog.xml
index 0989ec2199a..11437a2ce3a 100644
--- a/database/migration/src/main/resources/sql_metadata_changelog.xml
+++ b/database/migration/src/main/resources/sql_metadata_changelog.xml
@@ -19,6 +19,7 @@
+