From 3f9ce28f8403176e7cd2501c1d3abe54475e013e Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Thu, 14 Dec 2023 10:27:15 -0500 Subject: [PATCH] Revert "WX-1409 Java 17 (#7342)" This reverts commit 0ecbf53566bc4bbedc6c866aee2eaf044111e66e. --- .github/set_up_cromwell_action/action.yml | 10 +- .github/workflows/chart_update_on_merge.yml | 5 +- .github/workflows/docker_build_test.yml | 5 +- .github/workflows/integration_tests.yml | 8 - .github/workflows/trivy.yml | 5 +- .sdkmanrc | 2 +- CHANGELOG.md | 7 - .../api/DaemonizedDefaultThreadFactory.scala | 3 +- .../scala/cloud/nio/spi/CloudNioPath.scala | 2 +- .../main/scala/cloud/nio/spi/UnixPath.scala | 2 +- docs/Configuring.md | 2 +- docs/Releases.md | 4 +- docs/tutorials/FiveMinuteIntro.md | 6 +- project/Publishing.scala | 5 +- publish/docker-setup.sh | 2 +- runConfigurations/renderCiResources.run.xml | 4 +- scripts/docker-compose-mysql/README.md | 1 + .../compose/cromwell/Dockerfile | 9 + .../app-config/cromwell-application.conf | 61 +++ .../compose/mysql/init/init_user.sql | 4 + .../docker-compose-mysql/docker-compose.yml | 22 + .../jes-cromwell/docker-compose.yml | 9 + .../jes-cromwell/jes-config/application.conf | 73 +++ scripts/docker-develop/Dockerfile | 64 +++ scripts/docker-develop/README.md | 60 +++ src/ci/bin/test.inc.sh | 2 +- .../docker-compose/cromwell-test/Dockerfile | 2 +- .../cromwell-test/docker-setup.sh | 16 +- ...haredFileSystemJobExecutionActorSpec.scala | 430 ++++++++++++++++++ wom/src/main/resources/reference.conf | 2 +- .../test/scala/wom/util/YamlUtilsSpec.scala | 6 +- 31 files changed, 776 insertions(+), 57 deletions(-) create mode 100644 scripts/docker-compose-mysql/README.md create mode 100644 scripts/docker-compose-mysql/compose/cromwell/Dockerfile create mode 100644 scripts/docker-compose-mysql/compose/cromwell/app-config/cromwell-application.conf create mode 100644 scripts/docker-compose-mysql/compose/mysql/init/init_user.sql create mode 100644 scripts/docker-compose-mysql/docker-compose.yml create mode 100644 scripts/docker-compose-mysql/jes-cromwell/docker-compose.yml create mode 100644 scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf create mode 100644 scripts/docker-develop/Dockerfile create mode 100644 scripts/docker-develop/README.md create mode 100644 supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 1cbe834d2bd..13cb11601db 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -3,8 +3,7 @@ name: 'Set Up Cromwell Steps' description: Specific steps that will set up git secrets, java, sbt, and Cromwell on the local machine. inputs: - cromwell_repo_token: - description: "As an input to this action, you are required to pass in a token that can be used to authenticate while checking out Cromwell." + cromwell_repo_token: #As an input to this action, you are required to pass in a token that can be used to authenticate while checking out Cromwell. required: true runs: @@ -13,13 +12,13 @@ runs: #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - uses: actions/checkout@v3 name: Checkout Coursier Cache - - uses: coursier/cache-action@v6 + - uses: coursier/cache-action@v6 name: Enable Coursier Cache #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. - name: Git secrets setup run: | - git clone --quiet https://github.com/awslabs/git-secrets.git ~/git-secrets + git clone https://github.com/awslabs/git-secrets.git ~/git-secrets cd ~/git-secrets git checkout ad82d68ee924906a0401dfd48de5057731a9bc84 sudo make install @@ -44,4 +43,5 @@ runs: uses: actions/setup-java@v3 with: distribution: temurin - java-version: 17 + java-version: 11 + diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index b3f7199ab35..a2b14f2ec65 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -23,10 +23,9 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work path: cromwell - - uses: actions/setup-java@v4 + - uses: olafurpg/setup-scala@v10 with: - distribution: 'temurin' - java-version: '17' + java-version: adopt@1.11 - name: Clone Cromwhelm uses: actions/checkout@v2 with: diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index a50e39a380f..01c2ea502c9 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -25,10 +25,9 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} path: cromwell - - uses: actions/setup-java@v4 + - uses: olafurpg/setup-scala@v10 with: - distribution: 'temurin' - java-version: '17' + java-version: adopt@1.11 # The following invocation should be as similar as possible to the one in chart_update_on_merge.yml # To state the obvious: This test should not publish anything. It should simply verify that the build completes. - name: Build Cromwell Docker diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 028b53db130..61ed3c5af41 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -104,14 +104,6 @@ jobs: set -e echo Running test.sh ./src/ci/bin/test.sh - # If a build step fails, activate SSH and idle for 60 minutes - # - name: Setup tmate session - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # timeout-minutes: 60 - # with: - # limit-access-to-actor: true - # detached: true # always() is some github magic that forces the following step to run, even when the previous fails. # Without it, the if statement won't be evaluated on a test failure. - uses: ravsamhq/notify-slack-action@v2 diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 69a90937671..b005da65041 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -20,10 +20,9 @@ jobs: - uses: actions/checkout@v2 # fetch SBT package - - uses: actions/setup-java@v4 + - uses: olafurpg/setup-scala@v10 with: - distribution: 'temurin' - java-version: '17' + java-version: adopt@1.11 # set up SBT cache - uses: actions/cache@v2 diff --git a/.sdkmanrc b/.sdkmanrc index f3b37566860..0262a261019 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,3 +1,3 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below -java=17.0.9-tem +java=11.0.11.hs-adpt diff --git a/CHANGELOG.md b/CHANGELOG.md index d670e74a43e..a581852c02e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,5 @@ # Cromwell Change Log -## 87 Release Notes - -### Java 17 - -As of this version, a distribution of Java 17 is required to run Cromwell. Cromwell is developed, tested, and -containerized using [Eclipse Temurin](https://adoptium.net/temurin/releases/?version=17). - ## 86 Release Notes ### GCP Batch diff --git a/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala b/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala index d75299aa590..db9eb07d79b 100644 --- a/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala +++ b/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala @@ -14,7 +14,8 @@ import java.util.concurrent.atomic.AtomicInteger * > main thread). */ object DaemonizedDefaultThreadFactory extends ThreadFactory { - private val group = Thread.currentThread.getThreadGroup + private val s = System.getSecurityManager + private val group = if (s != null) s.getThreadGroup else Thread.currentThread.getThreadGroup private val threadNumber = new AtomicInteger(1) private val namePrefix = "daemonpool-thread-" diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala index 530ba5526ef..b6803670d39 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala @@ -176,7 +176,7 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un ): WatchKey = throw new UnsupportedOperationException override def iterator(): java.util.Iterator[Path] = - if (unixPath.izEmpty || unixPath.isRoot) { + if (unixPath.isEmpty || unixPath.isRoot) { java.util.Collections.emptyIterator() } else { unixPath.split().to(LazyList).map(part => newPath(UnixPath.getPath(part)).asInstanceOf[Path]).iterator.asJava diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala index 7d929d148ab..b7d9ed02035 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala @@ -67,7 +67,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def isAbsolute: Boolean = UnixPath.isAbsolute(path) - def izEmpty: Boolean = path.isEmpty + def isEmpty: Boolean = path.isEmpty def hasTrailingSeparator: Boolean = UnixPath.hasTrailingSeparator(path) diff --git a/docs/Configuring.md b/docs/Configuring.md index 288ec9723de..aa810d19a0c 100644 --- a/docs/Configuring.md +++ b/docs/Configuring.md @@ -689,7 +689,7 @@ This limit may be configured via the configuration value: ```hocon yaml { - max-depth = 100 + max-depth = 1000 } ``` diff --git a/docs/Releases.md b/docs/Releases.md index a6a709518b9..9699d7d44bc 100644 --- a/docs/Releases.md +++ b/docs/Releases.md @@ -19,8 +19,8 @@ Mac users with Homebrew can also get Cromwell with the command `brew install cro This documentation frequently refers to a "Cromwell jar" with a name like `cromwell-.jar`. This is the main artifact in Cromwell releases that contains all executable Cromwell code and default configuration. -A distribution of Java 17 is required to run Cromwell. Cromwell is developed, tested, and containerized using -[AdoptOpenJDK 17 HotSpot](https://adoptopenjdk.net/). +A distribution of Java 11 is required to run Cromwell. Cromwell is developed, tested, and containerized using +[AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/). For users running a Cromwell server [a docker image](https://hub.docker.com/r/broadinstitute/cromwell) has been made available. diff --git a/docs/tutorials/FiveMinuteIntro.md b/docs/tutorials/FiveMinuteIntro.md index f15e28087b9..1c16a56c606 100644 --- a/docs/tutorials/FiveMinuteIntro.md +++ b/docs/tutorials/FiveMinuteIntro.md @@ -3,11 +3,11 @@ ### Prerequisites: * A Unix-based operating system (yes, that includes Mac!) -* A Java 17 runtime environment +* A Java 11 runtime environment * You can see what you have by running `$ java -version` on a terminal. * If not, consider installing via conda or brew [as explained here](../Releases.md). - * We recommend [SDKMAN](https://sdkman.io/install) to install the latest 17 build of [Temurin](https://adoptium.net/temurin/releases/?version=17) - * `sdk install 17.0.9-tem` as of the time of this writing + * We recommend [SDKMAN](https://sdkman.io/install) to install the latest 11 build of [Temurin](https://adoptium.net/temurin/releases/?version=11) + * `sdk install java 11.0.16-tem` as of the time of this writing * You might need to update the `export JAVA_HOME` in your bash profile to point to your JAVA install location. * A sense of adventure! diff --git a/project/Publishing.scala b/project/Publishing.scala index c8a601a4f0c..0af27e77147 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -63,7 +63,7 @@ object Publishing { val additionalDockerInstr: Seq[Instruction] = (dockerCustomSettings ?? Nil).value new Dockerfile { - from("us.gcr.io/broad-dsp-gcr-public/base/jre:17-debian") + from("us.gcr.io/broad-dsp-gcr-public/base/jre:11-debian") expose(8000) add(artifact, artifactTargetPath) runRaw(s"ln -s $artifactTargetPath /app/$projectName.jar") @@ -163,7 +163,8 @@ object Publishing { val additionalResolvers = List( broadArtifactoryResolver, broadArtifactoryResolverSnap, - ) ++ Resolver.sonatypeOssRepos("releases") + Resolver.sonatypeRepo("releases") + ) private val artifactoryCredentialsFile = file("target/ci/resources/artifactory_credentials.properties").getAbsoluteFile diff --git a/publish/docker-setup.sh b/publish/docker-setup.sh index bc39f31d10a..613d251df96 100755 --- a/publish/docker-setup.sh +++ b/publish/docker-setup.sh @@ -20,7 +20,7 @@ mkdir -p /etc/apt/keyrings wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list apt update -apt install -y temurin-17-jdk +apt install -y temurin-11-jdk # Install jq 1.6 to ensure --rawfile is supported curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/bin/jq diff --git a/runConfigurations/renderCiResources.run.xml b/runConfigurations/renderCiResources.run.xml index ef56bf5e4fd..2a5ab5e29b8 100644 --- a/runConfigurations/renderCiResources.run.xml +++ b/runConfigurations/renderCiResources.run.xml @@ -7,8 +7,8 @@