From 0026944597bc9c465af9faeb36bffffa9bcc31dc Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 20 Nov 2024 02:33:06 +0100 Subject: [PATCH 1/6] Allow users to customize coursier ResolutionParams (#3949) This adds a `JavaModule#resolutionParams` task, allowing users to customize coursier's [`ResolutionParams`](https://github.com/coursier/coursier/blob/b04937dda13cee80b3743c84f471a69eb430047e/modules/coursier/shared/src/main/scala/coursier/params/ResolutionParams.scala). These allow to customize some aspects of dependency resolution. In particular, `ResolutionParams#enableDependencyOverrides`, when set to `Some(false)`, disables the recent BOM changes added (and enabled by default) in coursier `2.1.17`. This might be useful, if ever users aren't happy with the versions it selects, or want to wait a bit before enabling that, or if they find out performance suffers because of it for them. The new task is anonymous (it's not a target, it's not serialized on disk), as `ResolutionParams` has a field that doesn't easily lend itself to serialization (`rules`, because of the `Rule` type, which is an open abstract class). --- main/util/src/mill/util/CoursierSupport.scala | 69 ++++++++++++++++--- .../src/mill/scalalib/CoursierModule.scala | 54 ++++++++++++++- scalalib/src/mill/scalalib/JavaModule.scala | 3 +- scalalib/src/mill/scalalib/Lib.scala | 62 +++++++++++++++-- .../scalalib/CoursierParametersTests.scala | 44 ++++++++++++ 5 files changed, 215 insertions(+), 17 deletions(-) create mode 100644 scalalib/test/src/mill/scalalib/CoursierParametersTests.scala diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index ba09b7bfb0b..13e59bb78f3 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -45,7 +45,8 @@ trait CoursierSupport { ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, resolveFilter: os.Path => Boolean = _ => true, - artifactTypes: Option[Set[Type]] = None + artifactTypes: Option[Set[Type]] = None, + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { val org = dep.module.organization.value @@ -75,7 +76,8 @@ trait CoursierSupport { mapDependencies, customizer, ctx, - coursierCacheCustomizer + coursierCacheCustomizer, + resolutionParams ) resolutionRes.flatMap { resolution => @@ -114,6 +116,33 @@ trait CoursierSupport { } } + // bin-compat shim + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[Dependency], + force: IterableOnce[Dependency], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[Resolution => Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]], + resolveFilter: os.Path => Boolean, + artifactTypes: Option[Set[Type]] + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + force, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + resolveFilter, + artifactTypes, + ResolutionParams() + ) + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDependencies( repositories: Seq[Repository], @@ -135,7 +164,8 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - resolveFilter + resolveFilter, + None ) @deprecated( @@ -159,7 +189,8 @@ trait CoursierSupport { mapDependencies, customizer, ctx, - coursierCacheCustomizer + coursierCacheCustomizer, + ResolutionParams() ) (deps0, res.getOrThrow) } @@ -171,7 +202,8 @@ trait CoursierSupport { mapDependencies: Option[Dependency => Dependency] = None, customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, - coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Resolution] = { val rootDeps = deps.iterator @@ -185,14 +217,14 @@ trait CoursierSupport { val coursierCache0 = coursierCache(ctx, coursierCacheCustomizer) - val resolutionParams = ResolutionParams() - .withForceVersion(forceVersions) + val resolutionParams0 = resolutionParams + .addForceVersion(forceVersions.toSeq: _*) val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) .withRepositories(repositories) - .withResolutionParams(resolutionParams) + .withResolutionParams(resolutionParams0) .withMapDependenciesOpt(mapDependencies) resolve.either() match { @@ -230,6 +262,27 @@ trait CoursierSupport { } } + // bin-compat shim + def resolveDependenciesMetadataSafe( + repositories: Seq[Repository], + deps: IterableOnce[Dependency], + force: IterableOnce[Dependency], + mapDependencies: Option[Dependency => Dependency], + customizer: Option[Resolution => Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] + ): Result[Resolution] = + resolveDependenciesMetadataSafe( + repositories, + deps, + force, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + } object CoursierSupport { diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 0de8950488f..e195a402d32 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -1,6 +1,7 @@ package mill.scalalib import coursier.cache.FileCache +import coursier.params.ResolutionParams import coursier.{Dependency, Repository, Resolve, Type} import coursier.core.Resolution import mill.define.Task @@ -39,7 +40,8 @@ trait CoursierModule extends mill.Module { mapDependencies = Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), - ctx = Some(implicitly[mill.api.Ctx.Log]) + ctx = Some(implicitly[mill.api.Ctx.Log]), + resolutionParams = resolutionParams() ) } @@ -131,6 +133,27 @@ trait CoursierModule extends mill.Module { : Task[Option[FileCache[coursier.util.Task] => FileCache[coursier.util.Task]]] = Task.Anon { None } + /** + * Resolution parameters, allowing to customize resolution internals + * + * This rarely needs to be changed. This allows to disable the new way coursier handles + * BOMs since coursier 2.1.17 (used in Mill since 0.12.3) for example, with: + * {{{ + * def resolutionParams = super.resolutionParams() + * .withEnableDependencyOverrides(Some(false)) + * }}} + * + * Note that versions forced with `Dep#forceVersion()` take over forced versions manually + * set in `resolutionParams`. The former should be favored to force versions in dependency + * resolution. + * + * The Scala version set via `ScalaModule#scalaVersion` also takes over any Scala version + * provided via `ResolutionParams#scalaVersionOpt`. + */ + def resolutionParams: Task[ResolutionParams] = Task.Anon { + ResolutionParams() + } + } object CoursierModule { @@ -142,9 +165,33 @@ object CoursierModule { ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[ coursier.cache.FileCache[coursier.util.Task] => coursier.cache.FileCache[coursier.util.Task] - ] = None + ] = None, + resolutionParams: ResolutionParams = ResolutionParams() ) { + // bin-compat shim + def this( + repositories: Seq[Repository], + bind: Dep => BoundDep, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[coursier.util.Task] => coursier.cache.FileCache[ + coursier.util.Task + ] + ] + ) = + this( + repositories, + bind, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean = false, @@ -158,7 +205,8 @@ object CoursierModule { mapDependencies = mapDependencies, customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, - ctx = ctx + ctx = ctx, + resolutionParams = resolutionParams ).getOrThrow } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 33172e36388..7a7da6ce7d2 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -885,7 +885,8 @@ trait JavaModule dependencies, Some(mapDependencies()), customizer = resolutionCustomizer(), - coursierCacheCustomizer = coursierCacheCustomizer() + coursierCacheCustomizer = coursierCacheCustomizer(), + resolutionParams = resolutionParams() ).getOrThrow val roots = whatDependsOn match { diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 07a10461524..cf4fd6330d8 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -1,6 +1,7 @@ package mill package scalalib +import coursier.params.ResolutionParams import coursier.util.Task import coursier.{Dependency, Repository, Resolution, Type} import mill.api.{Ctx, Loose, PathRef, Result} @@ -59,7 +60,8 @@ object Lib { ctx: Option[Ctx.Log] = None, coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] - ] = None + ] = None, + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Resolution] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependenciesMetadataSafe( @@ -69,10 +71,32 @@ object Lib { mapDependencies = mapDependencies, customizer = customizer, ctx = ctx, - coursierCacheCustomizer = coursierCacheCustomizer + coursierCacheCustomizer = coursierCacheCustomizer, + resolutionParams = resolutionParams ) } + // bin-compat shim + def resolveDependenciesMetadataSafe( + repositories: Seq[Repository], + deps: IterableOnce[BoundDep], + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] + ] + ): Result[Resolution] = + resolveDependenciesMetadataSafe( + repositories, + deps, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + /** * Resolve dependencies using Coursier. * @@ -90,7 +114,8 @@ object Lib { coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, - artifactTypes: Option[Set[Type]] = None + artifactTypes: Option[Set[Type]] = None, + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( @@ -102,10 +127,36 @@ object Lib { mapDependencies = mapDependencies, customizer = customizer, ctx = ctx, - coursierCacheCustomizer = coursierCacheCustomizer + coursierCacheCustomizer = coursierCacheCustomizer, + resolutionParams = resolutionParams ).map(_.map(_.withRevalidateOnce)) } + // bin-compat shim + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[BoundDep], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] + ], + artifactTypes: Option[Set[Type]] + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + artifactTypes, + ResolutionParams() + ) + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDependencies( repositories: Seq[Repository], @@ -126,7 +177,8 @@ object Lib { customizer, ctx, coursierCacheCustomizer, - None + None, + ResolutionParams() ) def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String): Loose.Agg[Dep] = diff --git a/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala b/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala new file mode 100644 index 00000000000..49f33484f10 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala @@ -0,0 +1,44 @@ +package mill.scalalib + +import coursier.util.StringInterpolators.SafeModule +import mill.Agg +import mill.define.Task +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import utest._ + +object CoursierParametersTests extends TestSuite { + + val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "coursier" + + object CoursierTest extends TestBaseModule { + object core extends ScalaModule { + def scalaVersion = "2.13.12" + def ivyDeps = Task { + Agg(ivy"com.lihaoyi::pprint:0.9.0") + } + def resolutionParams = Task.Anon { + super.resolutionParams() + .addForceVersion((mod"com.lihaoyi:pprint_2.13", "0.8.1")) + } + } + } + + def tests: Tests = Tests { + test("coursierParams") - UnitTester(CoursierTest, null).scoped { eval => + val Right(result) = eval.apply(CoursierTest.core.compileClasspath) + val classPath = result.value.toSeq.map(_.path) + val pprintVersion = classPath + .map(_.last) + .filter(_.endsWith(".jar")) + .filter(_.startsWith("pprint_2.13-")) + .map(_.stripPrefix("pprint_2.13-").stripSuffix(".jar")) + .headOption + .getOrElse { + sys.error(s"pprint not found in class path $classPath") + } + val expectedPprintVersion = "0.8.1" + assert(pprintVersion == expectedPprintVersion) + } + } +} From 1a045850ad4dfb80d30253b571b7279bb9ea5825 Mon Sep 17 00:00:00 2001 From: Georg Ofenbeck Date: Wed, 20 Nov 2024 02:34:03 +0100 Subject: [PATCH 2/6] align SonatypeCentralPublishModule with SonatypePublisher (#3987) In the current version the SonatypeCentralPublishModule has non intented behaviour with PublishModule.defaultGpgArgs. It will default to the args without a passphrase and afterwards ignore the passphrase env. var as a gpgarg is already set. To get the "correct" behaviour one has to override the args with an empty string to then pick up the "PGP_PASSPHRASE" env. arg. Also changed the env. name to match the one from SonaTypePublisher "PGP_PASSPHRASE" -> "MILL_PGP_PASSPHRASE". Currently pgp and gpg are getting used both in terms of naming - guessing going for gpg in the future might be more clear. Co-authored-by: Georg Ofenbeck - taaofge1 --- contrib/sonatypecentral/readme.adoc | 4 ++-- .../SonatypeCentralPublishModule.scala | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/contrib/sonatypecentral/readme.adoc b/contrib/sonatypecentral/readme.adoc index 9ee00bbe145..27ea99b07d4 100644 --- a/contrib/sonatypecentral/readme.adoc +++ b/contrib/sonatypecentral/readme.adoc @@ -29,7 +29,7 @@ $ mill -i \ mill.contrib.sonatypecentral.SonatypeCentralPublishModule/publishAll \ --username myusername \ --password mypassword \ ---gpgArgs --passphrase=$GPG_PASSPHRASE,--no-tty,--pinentry-mode,loopback,--batch,--yes,-a,-b \ +--gpgArgs --passphrase=$MILL_PGP_PASSPHRASE,--no-tty,--pinentry-mode,loopback,--batch,--yes,-a,-b \ --publishArtifacts __.publishArtifacts \ --readTimeout 36000 \ --awaitTimeout 36000 \ @@ -68,7 +68,7 @@ The `mill.contrib.sonatypecentral.SonatypeCentralPublishModule/publishAll` metho `password`: The password for calling the Sonatype Central publishing api. Defaults to the `SONATYPE_PASSWORD` environment variable if unset. If neither the parameter nor the environment variable are set, an error will be thrown. + -`gpgArgs`: Arguments to pass to the gpg package for signing artifacts. _Default: `--batch, --yes, -a, -b`._ + +`gpgArgs`: Arguments to pass to the gpg package for signing artifacts. Uses the `MILL_PGP_PASSPHRASE` environment variable if set. _Default: `[--passphrase=$MILL_PGP_PASSPHRASE], --no-tty, --pinentry-mode, loopback, --batch, --yes, -a, -b`._ + `publishArtifacts`: The command for generating all publishable artifacts (ex. `__.publishArtifacts`). Required. + diff --git a/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala b/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala index 8896dd859c7..fd5fc26f386 100644 --- a/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala +++ b/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala @@ -13,7 +13,6 @@ import mill.contrib.sonatypecentral.SonatypeCentralPublishModule.{ getPublishingTypeFromReleaseFlag, getSonatypeCredentials } -import mill.scalalib.PublishModule.defaultGpgArgs import mill.scalalib.publish.Artifact import mill.scalalib.publish.SonatypeHelpers.{ PASSWORD_ENV_VARIABLE_NAME, @@ -21,7 +20,9 @@ import mill.scalalib.publish.SonatypeHelpers.{ } trait SonatypeCentralPublishModule extends PublishModule { - def sonatypeCentralGpgArgs: T[String] = Task { defaultGpgArgs.mkString(",") } + def sonatypeCentralGpgArgs: T[String] = Task { + PublishModule.defaultGpgArgsForPassphrase(T.env.get("MILL_PGP_PASSPHRASE")).mkString(",") + } def sonatypeCentralConnectTimeout: T[Int] = Task { defaultConnectTimeout } @@ -43,10 +44,7 @@ trait SonatypeCentralPublishModule extends PublishModule { PublishModule.pgpImportSecretIfProvided(T.env) val publisher = new SonatypeCentralPublisher( credentials = finalCredentials, - gpgArgs = sonatypeCentralGpgArgs() match { - case "" => PublishModule.defaultGpgArgsForPassphrase(T.env.get("PGP_PASSPHRASE")) - case gpgArgs => gpgArgs.split(",").toIndexedSeq - }, + gpgArgs = sonatypeCentralGpgArgs().split(",").toIndexedSeq, connectTimeout = sonatypeCentralConnectTimeout(), readTimeout = sonatypeCentralReadTimeout(), log = T.log, @@ -75,7 +73,7 @@ object SonatypeCentralPublishModule extends ExternalModule { username: String = defaultCredentials, password: String = defaultCredentials, shouldRelease: Boolean = defaultShouldRelease, - gpgArgs: String = defaultGpgArgs.mkString(","), + gpgArgs: String = "", readTimeout: Int = defaultReadTimeout, connectTimeout: Int = defaultConnectTimeout, awaitTimeout: Int = defaultAwaitTimeout, @@ -93,7 +91,7 @@ object SonatypeCentralPublishModule extends ExternalModule { val publisher = new SonatypeCentralPublisher( credentials = finalCredentials, gpgArgs = gpgArgs match { - case "" => PublishModule.defaultGpgArgsForPassphrase(T.env.get("PGP_PASSPHRASE")) + case "" => PublishModule.defaultGpgArgsForPassphrase(T.env.get("MILL_PGP_PASSPHRASE")) case gpgArgs => gpgArgs.split(",").toIndexedSeq }, connectTimeout = connectTimeout, From a458d471abd9b9a00135e8c5fa134b8870f226a2 Mon Sep 17 00:00:00 2001 From: albertpchen Date: Wed, 20 Nov 2024 11:09:27 +0800 Subject: [PATCH 3/6] Make JVM version configurable per-module (#3716) Fixes https://github.com/com-lihaoyi/mill/issues/3480. Changes: - adds `coursier-jvm` as a dependency to the `util` module, this library is for fetching JVMs using coursier and the coursier [jvm-index](https://github.com/coursier/jvm-index) - add a `def javaHome: Task[Option[PathRef]]` task to `ZincWorkerModule`. This defaults to `None`, which is the existing behavior of using mill's java home. - Users who want to use a custom JVM need to define a custom `ZincWorkerModule` and override `def jvmId`, optionally `def jvmIndexVersion` - updates the `mockito` and `commons-io` examples to use the new configuration options. Now these examples should run even when mill itself is not running on java 11. - This also required adding `-encoding utf-8` to the projects' javac options. - update the `run` and `test` tasks to use the `zincWorker`'s java home if a different one specified Notes: * `JavaModule#compile` forks a new JVM for each compilation to support custom JVM versions, and `ScalaModule#compile` does not support custom JVM versions at all. This would require a deeper refactoring of `ZincWorkerModule` to fix * --------- Co-authored-by: Li Haoyi --- build.mill | 12 +- .../jmh/src/mill/contrib/jmh/JmhModule.scala | 4 +- docs/modules/ROOT/nav.adoc | 2 + .../configuring-jvm-versions.adoc | 8 + .../1-custom-jvms/bar/src/bar/Bar.scala | 7 + .../depth/javahome/1-custom-jvms/build.mill | 125 ++++++++++++ .../1-custom-jvms/foo/src/foo/Foo.java | 8 + .../foo/test/src/foo/FooTest.java | 13 ++ .../1-custom-jvms/qux/src/qux/Qux.java | 8 + example/package.mill | 2 +- example/thirdparty/commons-io/build.mill | 8 + example/thirdparty/mockito/build.mill | 8 + main/api/src/mill/api/JsonFormatters.scala | 8 +- main/package.mill | 6 +- main/util/src/mill/util/CoursierSupport.scala | 54 ++++++ main/util/src/mill/util/Jvm.scala | 107 +++++++++-- readme.adoc | 8 + scalalib/package.mill | 3 +- scalalib/src/mill/scalalib/JavaModule.scala | 36 +++- scalalib/src/mill/scalalib/RunModule.scala | 26 ++- scalalib/src/mill/scalalib/ScalaModule.scala | 2 +- scalalib/src/mill/scalalib/TestModule.scala | 38 ++-- .../src/mill/scalalib/TestModuleUtil.scala | 46 +++-- .../src/mill/scalalib/ZincWorkerModule.scala | 34 +++- .../resources/hello-java/core/src/Core.java | 13 +- .../hello-java/core/test/src/MyCoreTests.java | 21 +- .../java-scala-11/javamodule/src/Main.java | 14 ++ .../java-scala-11/scalamodule/src/Main.scala | 14 ++ .../src/mill/scalalib/HelloJavaTests.scala | 14 +- .../src/mill/scalalib/JavaHomeTests.scala | 181 ++++++++++++++++++ .../worker/DiscoverMainClassesMain.scala | 84 ++++++++ .../mill/scalalib/worker/ZincWorkerImpl.scala | 90 +-------- .../mill/testrunner/DiscoverTestsMain.scala | 36 ++++ .../mill/testrunner/GetTestTasksMain.scala | 49 +++++ 34 files changed, 930 insertions(+), 159 deletions(-) create mode 100644 docs/modules/ROOT/pages/fundamentals/configuring-jvm-versions.adoc create mode 100644 example/depth/javahome/1-custom-jvms/bar/src/bar/Bar.scala create mode 100644 example/depth/javahome/1-custom-jvms/build.mill create mode 100644 example/depth/javahome/1-custom-jvms/foo/src/foo/Foo.java create mode 100644 example/depth/javahome/1-custom-jvms/foo/test/src/foo/FooTest.java create mode 100644 example/depth/javahome/1-custom-jvms/qux/src/qux/Qux.java create mode 100644 scalalib/test/resources/java-scala-11/javamodule/src/Main.java create mode 100644 scalalib/test/resources/java-scala-11/scalamodule/src/Main.scala create mode 100644 scalalib/test/src/mill/scalalib/JavaHomeTests.scala create mode 100644 scalalib/worker/src/mill/scalalib/worker/DiscoverMainClassesMain.scala create mode 100644 testrunner/src/mill/testrunner/DiscoverTestsMain.scala create mode 100644 testrunner/src/mill/testrunner/GetTestTasksMain.scala diff --git a/build.mill b/build.mill index f5e604449c2..99565ab6818 100644 --- a/build.mill +++ b/build.mill @@ -116,8 +116,10 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursier = ivy"io.get-coursier::coursier:2.1.16" + val coursierVersion = "2.1.16" + val coursier = ivy"io.get-coursier::coursier:$coursierVersion" val coursierInterface = ivy"io.get-coursier:interface:1.0.22" + val coursierJvm = ivy"io.get-coursier::coursier-jvm:$coursierVersion" val cask = ivy"com.lihaoyi::cask:0.9.4" val castor = ivy"com.lihaoyi::castor:0.3.0" @@ -192,6 +194,7 @@ object Deps { val mavenResolverTransportFile = ivy"org.apache.maven.resolver:maven-resolver-transport-file:$mavenResolverVersion" val mavenResolverTransportHttp = ivy"org.apache.maven.resolver:maven-resolver-transport-http:$mavenResolverVersion" val mavenResolverTransportWagon = ivy"org.apache.maven.resolver:maven-resolver-transport-wagon:$mavenResolverVersion" + val coursierJvmIndexVersion = "0.0.4-70-51469f" object RuntimeDeps { val dokkaVersion = "1.9.20" @@ -567,6 +570,13 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { // https://github.com/com-lihaoyi/mill/pull/3503 ProblemFilter.exclude[ReversedMissingMethodProblem]( "mill.scalalib.ScalaModule#ScalaTests.mill$scalalib$ScalaModule$ScalaTests$$super$mandatoryScalacOptions" + ), + // Not sure why Mima is complaining when these are internal and private + ProblemFilter.exclude[Problem]( + "*.bspJvmBuildTarget" + ), + ProblemFilter.exclude[Problem]( + "mill.scalalib.RunModule#RunnerImpl.*" ) ) def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions diff --git a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala index 593d9a9cbef..d32ee5e6b09 100644 --- a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala +++ b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala @@ -43,7 +43,8 @@ trait JmhModule extends JavaModule { classPath = (runClasspath() ++ generatorDeps()).map(_.path) ++ Seq(compileGeneratedSources().path, resources), mainArgs = args, - workingDir = T.ctx().dest + workingDir = T.ctx().dest, + javaHome = zincWorker().javaHome().map(_.path) ) } @@ -90,6 +91,7 @@ trait JmhModule extends JavaModule { resourcesDir.toString, "default" ), + javaHome = zincWorker().javaHome().map(_.path), jvmArgs = forkedArgs ) diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index dc3fbf12c11..37fe0701af1 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -11,6 +11,7 @@ ** xref:javalib/publishing.adoc[] ** xref:javalib/build-examples.adoc[] ** xref:javalib/web-examples.adoc[] + * xref:scalalib/intro.adoc[] ** xref:scalalib/module-config.adoc[] ** xref:scalalib/dependencies.adoc[] @@ -56,6 +57,7 @@ ** xref:fundamentals/library-deps.adoc[] ** xref:fundamentals/cross-builds.adoc[] ** xref:fundamentals/bundled-libraries.adoc[] +** xref:fundamentals/configuring-jvm-versions.adoc[] // This section talks about Mill plugins. While it could theoretically fit in // either section above, it is probably an important enough topic it is worth // breaking out on its own diff --git a/docs/modules/ROOT/pages/fundamentals/configuring-jvm-versions.adoc b/docs/modules/ROOT/pages/fundamentals/configuring-jvm-versions.adoc new file mode 100644 index 00000000000..8ebcd577640 --- /dev/null +++ b/docs/modules/ROOT/pages/fundamentals/configuring-jvm-versions.adoc @@ -0,0 +1,8 @@ += Configuring JVM Versions + +By default, Mill uses the same JVM that it itself is running on to compile/test/run +Java/Scala/Kotlin modules. This page goes into more detail about downloading +and using a custom Java home on a per-module basis. + +include::partial$example/depth/javahome/1-custom-jvms.adoc[] + diff --git a/example/depth/javahome/1-custom-jvms/bar/src/bar/Bar.scala b/example/depth/javahome/1-custom-jvms/bar/src/bar/Bar.scala new file mode 100644 index 00000000000..9ab3da6e1e8 --- /dev/null +++ b/example/depth/javahome/1-custom-jvms/bar/src/bar/Bar.scala @@ -0,0 +1,7 @@ +package bar + +object Bar { + def main(args: Array[String]): Unit = { + println(s"Bar running on Java ${sys.props("java.version")}") + } +} diff --git a/example/depth/javahome/1-custom-jvms/build.mill b/example/depth/javahome/1-custom-jvms/build.mill new file mode 100644 index 00000000000..c3f54325894 --- /dev/null +++ b/example/depth/javahome/1-custom-jvms/build.mill @@ -0,0 +1,125 @@ +// :link-jvm-indices: https://github.com/coursier/jvm-index/blob/master/indices/linux-arm64.json +// To override the default Java home, you can create a custom +// `ZincWorkerModule` and override the `zincWorker` method in your +// `JavaModule` or `ScalaModule` by pointing it to that custom object: + +// == Downloading Jvm By Version +// +// The `ZincWorkerModule.ForJvm` class takes a Jvm dependency string as its +// contructor argument and configures the module to fetch and cache the Jvm +// using coursier and use it for the compile, run, and test tasks. +// +// The string has the form: +// +// ---- +// "{name}:{version}" +// ---- +// +// To see what Jvms are available for download look at the index for your os +// and architecture {link-jvm-indices}[here]. +import mill._, javalib._ +import mill.define.ModuleRef + +object ZincWorkerJava11 extends ZincWorkerModule { + def jvmId = "temurin:18.0.2" +} + +object foo extends JavaModule { + def zincWorker = ModuleRef(ZincWorkerJava11) + + object test extends JavaTests with TestModule.Junit4 +} + +/** Usage + +> mill foo.run +Foo running on Java 18.0.2 + +> mill foo.test +Testing with JVM version: 18.0.2 +Test foo.FooTest.testSimple finished... + +*/ + +// Selecting a custom JVM via `ZincWorkerModule.ForJvm` means that JVM is used for +// compiling, testing, and running that module via Mill. Note that `.assembly` is not +// affected, as JVM assembly jars do not bundle a JVM and have to be run using a +// JVM installed on the target host machine. Configuration is done via `ZincWorkerModule` +// as the https://github.com/sbt/zinc[Zinc] incremental compiler is used for compiling +// Java and Scala sources. +// +// == Selecting JVM Index Versions +// + +// +// By default, Mill comes bundled with a version of the JVM index that was published when +// each version of Mill is released. This ensures that the JVM versions you pick are stable, +// but means that the latest JVM versions may not be available. You can pass in the JVM +// index version explicitly via `def jvmIndexVersion` below, choosing a published +// index version from the Maven Central (https://repo1.maven.org/maven2/io/get-coursier/jvm/indices/index-linux-arm64/[link]) +// + +import scalalib._ + +object ZincWorkerJava11Latest extends ZincWorkerModule { + def jvmId = "temurin:23.0.1" + def jvmIndexVersion = "latest.release" +} + +object bar extends ScalaModule { + def scalaVersion = "2.13.12" + def zincWorker = ModuleRef(ZincWorkerJava11Latest) + +} + +/** Usage + +> mill bar.run +Bar running on Java 23.0.1 + +*/ + +// == Explicit JVM Download URLs +// +// You can also pass in the JVM download URL explicitly. Note that if you do so, you need +// to ensure yourself that you are downloading the appropriate JVM distribution for your +// operating system and CPU architecture. In the example below we switch between Mac/ARM and +// Linux/X64, but you may have additional cases if you need to support Windows or other +// OS/CPU combinations + +import kotlinlib._ + +object ZincWorkerJava11Url extends ZincWorkerModule { + def jvmId = + if (sys.props("os.name") == "Mac OS X") { + "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_aarch64_mac_hotspot_22.0.2_9.tar.gz" + } else { + "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz" + } + +} + +object qux extends KotlinModule { + def kotlinVersion = "2.0.20" + def zincWorker = ModuleRef(ZincWorkerJava11Url) +} + +/** Usage + +> mill qux.run +Qux running on Java 22.0.2 + +*/ + +// == Locally-Installed JVMs +// +// Lastly, you can point Mill at any JVM distribution installed locally on disk via: + +object ZincWorkerLocalJvm extends ZincWorkerModule { + def javaHome = Some(PathRef(os.Path("/my/java/home"), quick = true)) +} + +object baz extends JavaModule { + def zincWorker = ModuleRef(ZincWorkerLocalJvm) + +} diff --git a/example/depth/javahome/1-custom-jvms/foo/src/foo/Foo.java b/example/depth/javahome/1-custom-jvms/foo/src/foo/Foo.java new file mode 100644 index 00000000000..9a0f5b33577 --- /dev/null +++ b/example/depth/javahome/1-custom-jvms/foo/src/foo/Foo.java @@ -0,0 +1,8 @@ +package foo; + +public class Foo { + + public static void main(String[] args) throws Exception { + System.out.println("Foo running on Java " + System.getProperty("java.version")); + } +} diff --git a/example/depth/javahome/1-custom-jvms/foo/test/src/foo/FooTest.java b/example/depth/javahome/1-custom-jvms/foo/test/src/foo/FooTest.java new file mode 100644 index 00000000000..afcdcb9a051 --- /dev/null +++ b/example/depth/javahome/1-custom-jvms/foo/test/src/foo/FooTest.java @@ -0,0 +1,13 @@ +package foo; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class FooTest { + @Test + public void testSimple() { + System.out.println("Testing with JVM version: " + System.getProperty("java.version")); + assertEquals(System.getProperty("java.version"), "18.0.2"); + } +} diff --git a/example/depth/javahome/1-custom-jvms/qux/src/qux/Qux.java b/example/depth/javahome/1-custom-jvms/qux/src/qux/Qux.java new file mode 100644 index 00000000000..28a3ac28905 --- /dev/null +++ b/example/depth/javahome/1-custom-jvms/qux/src/qux/Qux.java @@ -0,0 +1,8 @@ +package qux; + +public class Qux { + + public static void main(String[] args) throws Exception { + System.out.println("Qux running on Java " + System.getProperty("java.version")); + } +} diff --git a/example/package.mill b/example/package.mill index e4b0165136d..f87b18c58b6 100644 --- a/example/package.mill +++ b/example/package.mill @@ -87,7 +87,7 @@ object `package` extends RootModule with Module { object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large")) object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox")) - + object javahome extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "javahome")) } object extending extends Module { diff --git a/example/thirdparty/commons-io/build.mill b/example/thirdparty/commons-io/build.mill index 91b6d73f31f..68694fb3f20 100644 --- a/example/thirdparty/commons-io/build.mill +++ b/example/thirdparty/commons-io/build.mill @@ -1,9 +1,17 @@ package build import mill._, javalib._, publish._ +import mill.define.ModuleRef import $ivy.`com.lihaoyi::mill-contrib-jmh:$MILL_VERSION` import contrib.jmh.JmhModule object `package` extends RootModule with PublishModule with MavenModule { + + object ZincWorkerJava11 extends ZincWorkerModule { + def jvmId = "temurin:11.0.24" + } + + override def zincWorker = ModuleRef(ZincWorkerJava11) + def javacOptions = Seq("-encoding", "UTF-8") def publishVersion = "2.17.0-SNAPSHOT" def pomSettings = PomSettings( diff --git a/example/thirdparty/mockito/build.mill b/example/thirdparty/mockito/build.mill index 2952aa906e1..f00044a7b13 100644 --- a/example/thirdparty/mockito/build.mill +++ b/example/thirdparty/mockito/build.mill @@ -1,5 +1,6 @@ package build import mill._, javalib._ +import mill.define.ModuleRef object libraries { @@ -37,6 +38,8 @@ object libraries { } trait MockitoModule extends MavenModule { + override def zincWorker = ModuleRef(build.ZincWorkerJava11) + def javacOptions = Seq("-encoding", "UTF-8") def testModuleDeps: Seq[JavaModule] = Nil def testIvyDeps: T[Agg[Dep]] = Agg.empty[Dep] def testRuntimeIvyDeps: T[Agg[Dep]] = Agg.empty[Dep] @@ -68,6 +71,11 @@ trait MockitoModule extends MavenModule { } object `package` extends RootModule with MockitoModule { + + object ZincWorkerJava11 extends ZincWorkerModule { + def jvmId = "temurin:11.0.24" + } + def compileIvyDeps = Agg( libraries.hamcrest, libraries.junit4, diff --git a/main/api/src/mill/api/JsonFormatters.scala b/main/api/src/mill/api/JsonFormatters.scala index bf1b448955f..b708070fcda 100644 --- a/main/api/src/mill/api/JsonFormatters.scala +++ b/main/api/src/mill/api/JsonFormatters.scala @@ -21,8 +21,14 @@ trait JsonFormatters { /** * Additional [[mainargs.TokensReader]] instance to teach it how to read Ammonite paths + * + * Should be replaced by `PathTokensReader2` but kept for binary compatibility */ - implicit def PathTokensReader: mainargs.TokensReader[os.Path] = JsonFormatters.PathTokensReader0 + implicit def PathTokensReader: mainargs.TokensReader[os.Path] = + JsonFormatters.PathTokensReader0 + + def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] = + JsonFormatters.PathTokensReader0 implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String] .bimap[os.Path]( diff --git a/main/package.mill b/main/package.mill index d1db4eb4114..f1e13d182e9 100644 --- a/main/package.mill +++ b/main/package.mill @@ -100,7 +100,11 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI object util extends build.MillStableScalaModule { def moduleDeps = Seq(api, client) - def ivyDeps = Agg(build.Deps.coursier, build.Deps.jline) + def ivyDeps = Agg( + build.Deps.coursier, + build.Deps.coursierJvm, + build.Deps.jline + ) } object define extends build.MillStableScalaModule { diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 13e59bb78f3..c44b32c3c65 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -5,6 +5,7 @@ import coursier.error.FetchError.DownloadingArtifacts import coursier.error.ResolutionError.CantDownloadModule import coursier.params.ResolutionParams import coursier.parse.RepositoryParser +import coursier.jvm.{JvmCache, JvmChannel, JvmIndex, JavaHome} import coursier.util.Task import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type} import mill.api.Loose.Agg @@ -12,6 +13,8 @@ import mill.api.{Ctx, PathRef, Result} import scala.collection.mutable import scala.util.chaining.scalaUtilChainingOps +import coursier.cache.ArchiveCache +import scala.util.control.NonFatal trait CoursierSupport { import CoursierSupport._ @@ -194,6 +197,57 @@ trait CoursierSupport { ) (deps0, res.getOrThrow) } + def jvmIndex( + ctx: Option[mill.api.Ctx.Log] = None, + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None + ): JvmIndex = { + val coursierCache0 = coursierCache(ctx, coursierCacheCustomizer) + jvmIndex0(ctx, coursierCacheCustomizer).unsafeRun()(coursierCache0.ec) + } + + def jvmIndex0( + ctx: Option[mill.api.Ctx.Log] = None, + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, + jvmIndexVersion: String = "latest.release" + ): Task[JvmIndex] = { + val coursierCache0 = coursierCache(ctx, coursierCacheCustomizer) + JvmIndex.load( + cache = coursierCache0, // the coursier.cache.Cache instance to use + repositories = Resolve().repositories, // repositories to use + indexChannel = JvmChannel.module( + JvmChannel.centralModule(), + version = jvmIndexVersion + ) // use new indices published to Maven Central + ) + } + + /** + * Resolve java home using Coursier. + * + * The id string has format "$DISTRIBUTION:$VERSION". e.g. graalvm-community:23.0.0 + */ + def resolveJavaHome( + id: String, + ctx: Option[mill.api.Ctx.Log] = None, + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, + jvmIndexVersion: String = "latest.release" + ): Result[os.Path] = { + val coursierCache0 = coursierCache(ctx, coursierCacheCustomizer) + val jvmCache = JvmCache() + .withArchiveCache( + ArchiveCache().withCache(coursierCache0) + ) + .withIndex(jvmIndex0(ctx, coursierCacheCustomizer, jvmIndexVersion)) + val javaHome = JavaHome() + .withCache(jvmCache) + try { + val file = javaHome.get(id).unsafeRun()(coursierCache0.ec) + Result.Success(os.Path(file)) + } catch { + case NonFatal(error) => + Result.Exception(error, new Result.OuterStack((new Exception).getStackTrace)) + } + } def resolveDependenciesMetadataSafe( repositories: Seq[Repository], diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index 267d0566a8d..797c26321fd 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -26,11 +26,12 @@ object Jvm extends CoursierSupport { mainArgs: Seq[String] = Seq.empty, workingDir: os.Path = null, streamOut: Boolean = true, - check: Boolean = true + check: Boolean = true, + javaHome: Option[os.Path] = None )(implicit ctx: Ctx): CommandResult = { val commandArgs = - Vector(javaExe) ++ + Vector(javaExe(javaHome)) ++ jvmArgs ++ Vector("-cp", classPath.iterator.mkString(java.io.File.pathSeparator), mainClass) ++ mainArgs @@ -38,6 +39,7 @@ object Jvm extends CoursierSupport { val workingDir1 = Option(workingDir).getOrElse(ctx.dest) os.makeDir.all(workingDir1) + mill.main.client.DebugLog.println(commandArgs.toString()) os.proc(commandArgs) .call( cwd = workingDir1, @@ -47,6 +49,33 @@ object Jvm extends CoursierSupport { ) } + /** + * Runs a JVM subprocess with the given configuration and returns a + * [[os.CommandResult]] with it's aggregated output and error streams + */ + def callSubprocess( + mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String], + envArgs: Map[String, String], + mainArgs: Seq[String], + workingDir: os.Path, + streamOut: Boolean, + check: Boolean + )(implicit ctx: Ctx): CommandResult = { + callSubprocess( + mainClass, + classPath, + jvmArgs, + envArgs, + mainArgs, + workingDir, + streamOut, + check, + None + ) + } + /** * Runs a JVM subprocess with the given configuration and returns a * [[os.CommandResult]] with it's aggregated output and error streams @@ -66,9 +95,10 @@ object Jvm extends CoursierSupport { /** * Resolves a tool to a path under the currently used JDK (if known). */ - def jdkTool(toolName: String): String = { - sys.props - .get("java.home") + def jdkTool(toolName: String, javaHome: Option[os.Path]): String = { + javaHome + .map(_.toString()) + .orElse(sys.props.get("java.home")) .map(h => if (isWin) new File(h, s"bin\\${toolName}.exe") else new File(h, s"bin/${toolName}") @@ -78,7 +108,11 @@ object Jvm extends CoursierSupport { } - def javaExe: String = jdkTool("java") + def jdkTool(toolName: String): String = jdkTool(toolName, None) + + def javaExe(javaHome: Option[os.Path]): String = jdkTool("java", javaHome) + + def javaExe: String = javaExe(None) def defaultBackgroundOutputs(outputDir: os.Path): Option[(ProcessOutput, ProcessOutput)] = Some((outputDir / "stdout.log", outputDir / "stderr.log")) @@ -108,7 +142,8 @@ object Jvm extends CoursierSupport { workingDir: os.Path = null, background: Boolean = false, useCpPassingJar: Boolean = false, - runBackgroundLogToConsole: Boolean = false + runBackgroundLogToConsole: Boolean = false, + javaHome: Option[os.Path] = None )(implicit ctx: Ctx): Unit = { runSubprocessWithBackgroundOutputs( mainClass, @@ -130,10 +165,35 @@ object Jvm extends CoursierSupport { ) ) } else Jvm.defaultBackgroundOutputs(ctx.dest), - useCpPassingJar + useCpPassingJar, + javaHome ) } + // bincompat shim + def runSubprocess( + mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String], + envArgs: Map[String, String], + mainArgs: Seq[String], + workingDir: os.Path, + background: Boolean, + useCpPassingJar: Boolean, + runBackgroundLogToConsole: Boolean + )(implicit ctx: Ctx): Unit = + runSubprocess( + mainClass, + classPath, + jvmArgs, + envArgs, + mainArgs, + workingDir, + background, + useCpPassingJar, + runBackgroundLogToConsole, + None + ) // bincompat shim def runSubprocess( mainClass: String, @@ -153,7 +213,8 @@ object Jvm extends CoursierSupport { mainArgs, workingDir, background, - useCpPassingJar + useCpPassingJar, + false ) /** @@ -180,7 +241,8 @@ object Jvm extends CoursierSupport { mainArgs: Seq[String] = Seq.empty, workingDir: os.Path = null, backgroundOutputs: Option[Tuple2[ProcessOutput, ProcessOutput]] = None, - useCpPassingJar: Boolean = false + useCpPassingJar: Boolean = false, + javaHome: Option[os.Path] = None )(implicit ctx: Ctx): Unit = { val cp = @@ -204,7 +266,7 @@ object Jvm extends CoursierSupport { Seq(mainClass) } else Seq.empty val args = - Vector(javaExe) ++ + Vector(javaExe(javaHome)) ++ jvmArgs ++ cpArgument ++ mainClassArgument ++ @@ -218,6 +280,29 @@ object Jvm extends CoursierSupport { runSubprocess(args, envArgs, workingDir) } + // bincompat shim + def runSubprocessWithBackgroundOutputs( + mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String], + envArgs: Map[String, String], + mainArgs: Seq[String], + workingDir: os.Path, + backgroundOutputs: Option[Tuple2[ProcessOutput, ProcessOutput]], + useCpPassingJar: Boolean + )(implicit ctx: Ctx): Unit = + runSubprocessWithBackgroundOutputs( + mainClass, + classPath, + jvmArgs, + envArgs, + mainArgs, + workingDir, + backgroundOutputs, + useCpPassingJar, + None + )(ctx) + /** * Runs a generic subprocess and waits for it to terminate. If process exited with non-zero code, exception * will be thrown. If you want to manually handle exit code, check [[runSubprocessWithResult]] diff --git a/readme.adoc b/readme.adoc index e7fbbe5b74d..bbe3d3f0e79 100644 --- a/readme.adoc +++ b/readme.adoc @@ -255,6 +255,14 @@ In case of troubles with caching and/or incremental compilation, you can always rm -rf out/ ---- +=== Autofix and Autoformatting + +To run all autofixes and autoformatters: + +```bash +./mill __.fix + mill.javalib.palantirformat.PalantirFormatModule/ + mill.scalalib.scalafmt.ScalafmtModule/ + mill.kotlinlib.ktlint.KtlintModule/ +```` + == Project Layout The Mill project is organized roughly as follows: diff --git a/scalalib/package.mill b/scalalib/package.mill index b061af417dc..27355c72ed7 100644 --- a/scalalib/package.mill +++ b/scalalib/package.mill @@ -71,7 +71,8 @@ object `package` extends RootModule with build.MillStableScalaModule { }, "Dependency to jupiter-interface" ), - BuildInfo.Value("errorProneVersion", build.Deps.RuntimeDeps.errorProneCore.version) + BuildInfo.Value("errorProneVersion", build.Deps.RuntimeDeps.errorProneCore.version), + BuildInfo.Value("coursierJvmIndexVersion", build.Deps.coursierJvmIndexVersion) ) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 7a7da6ce7d2..44efaf281f6 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -102,15 +102,19 @@ trait JavaModule mainClass() match { case Some(m) => Right(m) case None => - zincWorker().worker().discoverMainClasses(compile()) match { - case Seq() => Left("No main class specified or found") - case Seq(main) => Right(main) - case mains => - Left( - s"Multiple main classes found (${mains.mkString(",")}) " + - "please explicitly specify which one to use by overriding `mainClass` " + - "or using `runMain <...args>` instead of `run`" - ) + if (zincWorker().javaHome().isDefined) { + super[RunModule].finalMainClassOpt() + } else { + zincWorker().worker().discoverMainClasses(compile()) match { + case Seq() => Left("No main class specified or found") + case Seq(main) => Right(main) + case mains => + Left( + s"Multiple main classes found (${mains.mkString(",")}) " + + "please explicitly specify which one to use by overriding `mainClass` " + + "or using `runMain <...args>` instead of `run`" + ) + } } } } @@ -1127,14 +1131,26 @@ trait JavaModule ) @internal + @deprecated("Use bspJvmBuildTargetTask instead", "0.12.3") def bspJvmBuildTarget: JvmBuildTarget = JvmBuildTarget( javaHome = Option(System.getProperty("java.home")).map(p => BspUri(os.Path(p))), javaVersion = Option(System.getProperty("java.version")) ) + @internal + def bspJvmBuildTargetTask: Task[JvmBuildTarget] = Task.Anon { + JvmBuildTarget( + javaHome = zincWorker() + .javaHome() + .map(p => BspUri(p.path)) + .orElse(Option(System.getProperty("java.home")).map(p => BspUri(os.Path(p)))), + javaVersion = Option(System.getProperty("java.version")) + ) + } + @internal override def bspBuildTargetData: Task[Option[(String, AnyRef)]] = Task.Anon { - Some((JvmBuildTarget.dataKind, bspJvmBuildTarget)) + Some((JvmBuildTarget.dataKind, bspJvmBuildTargetTask())) } } diff --git a/scalalib/src/mill/scalalib/RunModule.scala b/scalalib/src/mill/scalalib/RunModule.scala index 02b603985db..a371830e1e7 100644 --- a/scalalib/src/mill/scalalib/RunModule.scala +++ b/scalalib/src/mill/scalalib/RunModule.scala @@ -44,7 +44,22 @@ trait RunModule extends WithZincWorker { def mainClass: T[Option[String]] = None def allLocalMainClasses: T[Seq[String]] = Task { - zincWorker().worker().discoverMainClasses(localRunClasspath().map(_.path)) + val classpath = localRunClasspath().map(_.path) + if (zincWorker().javaHome().isDefined) { + Jvm + .callSubprocess( + mainClass = "mill.scalalib.worker.DiscoverMainClassesMain", + classPath = zincWorker().classpath().map(_.path), + mainArgs = Seq(classpath.mkString(",")), + javaHome = zincWorker().javaHome().map(_.path), + streamOut = false + ) + .out + .lines() + + } else { + zincWorker().worker().discoverMainClasses(classpath) + } } def finalMainClassOpt: T[Either[String, String]] = Task { @@ -135,7 +150,8 @@ trait RunModule extends WithZincWorker { runClasspath().map(_.path), forkArgs(), forkEnv(), - runUseArgsFile() + runUseArgsFile(), + zincWorker().javaHome().map(_.path) ) } @@ -243,7 +259,8 @@ object RunModule { runClasspath: Seq[os.Path], forkArgs0: Seq[String], forkEnv0: Map[String, String], - useCpPassingJar0: Boolean + useCpPassingJar0: Boolean, + javaHome: Option[os.Path] ) extends Runner { def run( @@ -269,7 +286,8 @@ object RunModule { case Some(b) => b case None => useCpPassingJar0 }, - runBackgroundLogToConsole = runBackgroundLogToConsole + runBackgroundLogToConsole = runBackgroundLogToConsole, + javaHome = javaHome ) } } diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index e6ba39ee3d8..f88b7f5773a 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -592,7 +592,7 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => scalaBinaryVersion = ZincWorkerUtil.scalaBinaryVersion(scalaVersion()), platform = ScalaPlatform.JVM, jars = scalaCompilerClasspath().map(_.path.toNIO.toUri.toString).iterator.toSeq, - jvmBuildTarget = Some(bspJvmBuildTarget) + jvmBuildTarget = Some(bspJvmBuildTargetTask()) ) )) } diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index 7a32c438c95..8d9c4435dd0 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -3,7 +3,7 @@ package mill.scalalib import mill.api.{Ctx, PathRef, Result} import mill.define.{Command, Task, TaskModule} import mill.scalalib.bsp.{BspBuildTarget, BspModule} -import mill.testrunner.{Framework, TestArgs, TestResult, TestRunner, TestRunnerUtils} +import mill.testrunner.{Framework, TestArgs, TestResult, TestRunner} import mill.util.Jvm import mill.{Agg, T} @@ -41,21 +41,24 @@ trait TestModule def testFramework: T[String] def discoveredTestClasses: T[Seq[String]] = Task { - val classes = Jvm.inprocess( - runClasspath().map(_.path), - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = true, - cl => { - val framework = Framework.framework(testFramework())(cl) - val classes = TestRunnerUtils.discoverTests(cl, framework, testClasspath().map(_.path)) - classes.toSeq.map(_._1.getName()) - .map { - case s if s.endsWith("$") => s.dropRight(1) - case s => s - } - } - ) + val classes = if (zincWorker().javaHome().isDefined) { + Jvm.callSubprocess( + mainClass = "mill.testrunner.DiscoverTestsMain", + classPath = zincWorker().scalalibClasspath().map(_.path), + mainArgs = + runClasspath().flatMap(p => Seq("--runCp", p.path.toString())) ++ + testClasspath().flatMap(p => Seq("--testCp", p.path.toString())) ++ + Seq("--framework", testFramework()), + javaHome = zincWorker().javaHome().map(_.path), + streamOut = false + ).out.lines() + } else { + mill.testrunner.DiscoverTestsMain.main0( + runClasspath().map(_.path), + testClasspath().map(_.path), + testFramework() + ) + } classes.sorted } @@ -200,7 +203,8 @@ trait TestModule forkEnv(), testSandboxWorkingDir(), forkWorkingDir(), - testReportXml() + testReportXml(), + zincWorker().javaHome().map(_.path) ) } diff --git a/scalalib/src/mill/scalalib/TestModuleUtil.scala b/scalalib/src/mill/scalalib/TestModuleUtil.scala index b7a29bcbdcc..55a6595f326 100644 --- a/scalalib/src/mill/scalalib/TestModuleUtil.scala +++ b/scalalib/src/mill/scalalib/TestModuleUtil.scala @@ -2,7 +2,7 @@ package mill.scalalib import mill.api.{Ctx, PathRef, Result} import mill.main.client.EnvVars -import mill.testrunner.{Framework, TestArgs, TestResult, TestRunnerUtils} +import mill.testrunner.{TestArgs, TestResult, TestRunnerUtils} import mill.util.Jvm import mill.{Agg, T} import sbt.testing.Status @@ -28,7 +28,8 @@ private[scalalib] object TestModuleUtil { forkEnv: Map[String, String], testSandboxWorkingDir: Boolean, forkWorkingDir: os.Path, - testReportXml: Option[String] + testReportXml: Option[String], + javaHome: Option[os.Path] )(implicit ctx: mill.api.Ctx) = { val (jvmArgs, props: Map[String, String]) = loadArgsAndProps(useArgsFile, forkArgs) @@ -72,7 +73,8 @@ private[scalalib] object TestModuleUtil { envArgs = forkEnv ++ resourceEnv, mainArgs = Seq(testRunnerClasspathArg, argsFile.toString), workingDir = if (testSandboxWorkingDir) sandbox else forkWorkingDir, - useCpPassingJar = useArgsFile + useCpPassingJar = useArgsFile, + javaHome = javaHome ) if (!os.exists(outputPath)) Left(s"Test reporting Failed: ${outputPath} does not exist") @@ -93,24 +95,36 @@ private[scalalib] object TestModuleUtil { else { // If test grouping is enabled and multiple test groups are detected, we need to // run test discovery via the test framework's own argument parsing and filtering - // logic once in memory before we potentially fork off multiple test groups that will + // logic once before we potentially fork off multiple test groups that will // each do the same thing and then run tests. This duplication is necessary so we can // skip test groups that we know will be empty, which is important because even an empty // test group requires spawning a JVM which can take 1+ seconds to realize there are no // tests to run and shut down - val discoveredTests = Jvm.inprocess( - (runClasspath ++ testrunnerEntrypointClasspath).map(_.path), - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = false, - TestRunnerUtils.getTestTasks0( - Framework.framework(testFramework), + + val discoveredTests = if (javaHome.isDefined) { + Jvm.callSubprocess( + mainClass = "mill.testrunner.GetTestTasksMain", + classPath = scalalibClasspath.map(_.path), + mainArgs = + (runClasspath ++ testrunnerEntrypointClasspath).flatMap(p => + Seq("--runCp", p.path.toString) + ) ++ + testClasspath.flatMap(p => Seq("--testCp", p.path.toString)) ++ + Seq("--framework", testFramework) ++ + selectors.flatMap(s => Seq("--selectors", s)) ++ + args.flatMap(s => Seq("--args", s)), + javaHome = javaHome, + streamOut = false + ).out.lines().toSet + } else { + mill.testrunner.GetTestTasksMain.main0( + (runClasspath ++ testrunnerEntrypointClasspath).map(_.path), testClasspath.map(_.path), - args, - cls => globFilter(cls.getName), - _ - ) - ).toSet + testFramework, + selectors, + args + ).toSet + } filteredClassLists0.map(_.filter(discoveredTests)).filter(_.nonEmpty) } diff --git a/scalalib/src/mill/scalalib/ZincWorkerModule.scala b/scalalib/src/mill/scalalib/ZincWorkerModule.scala index eb2ebb6f6b1..28744565e93 100644 --- a/scalalib/src/mill/scalalib/ZincWorkerModule.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerModule.scala @@ -4,10 +4,10 @@ import coursier.Repository import mainargs.Flag import mill._ import mill.api.{Ctx, FixSizedCache, KeyedLockedCache, PathRef, Result} -import mill.define.{ExternalModule, Discover} +import mill.define.{Discover, ExternalModule, Task} import mill.scalalib.Lib.resolveDependencies import mill.scalalib.api.ZincWorkerUtil.{isBinaryBridgeAvailable, isDotty, isDottyOrScala3} -import mill.scalalib.api.{ZincWorkerApi, ZincWorkerUtil, Versions} +import mill.scalalib.api.{Versions, ZincWorkerApi, ZincWorkerUtil} import mill.util.Util.millProjectModule /** @@ -20,7 +20,11 @@ object ZincWorkerModule extends ExternalModule with ZincWorkerModule with Coursi /** * A module managing an in-memory Zinc Scala incremental compiler */ -trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: CoursierModule => +trait ZincWorkerModule extends mill.Module with OfflineSupportModule with CoursierModule { + def jvmId: mill.define.Target[String] = Task[String] { "" } + + def jvmIndexVersion: mill.define.Target[String] = + mill.scalalib.api.Versions.coursierJvmIndexVersion def classpath: T[Agg[PathRef]] = Task { millProjectModule("mill-scalalib-worker", repositoriesTask()) @@ -44,6 +48,24 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou def zincLogDebug: T[Boolean] = Task.Input(T.ctx().log.debugEnabled) + /** + * Optional custom Java Home for the ZincWorker to use + * + * If this value is None, then the ZincWorker uses the same Java used to run + * the current mill instance. + */ + def javaHome: T[Option[PathRef]] = Task { + Option(jvmId()).filter(_ != "").map { id => + val path = mill.util.Jvm.resolveJavaHome( + id = id, + coursierCacheCustomizer = coursierCacheCustomizer(), + ctx = Some(implicitly[mill.api.Ctx.Log]), + jvmIndexVersion = jvmIndexVersion() + ).getOrThrow + PathRef(path, quick = true) + } + } + def worker: Worker[ZincWorkerApi] = Task.Worker { val jobs = T.ctx() match { case j: Ctx.Jobs => j.jobs @@ -66,7 +88,8 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou classOf[(Agg[PathRef], String) => PathRef], // compilerJarNameGrep classOf[KeyedLockedCache[_]], // compilerCache classOf[Boolean], // compileToJar - classOf[Boolean] // zincLogDebug + classOf[Boolean], // zincLogDebug + classOf[Option[PathRef]] // javaHome ) .newInstance( Left(( @@ -83,7 +106,8 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou ZincWorkerUtil.grepJar(_, "scala-compiler", _, sources = false), new FixSizedCache(jobs), java.lang.Boolean.FALSE, - java.lang.Boolean.valueOf(zincLogDebug()) + java.lang.Boolean.valueOf(zincLogDebug()), + javaHome() ) instance.asInstanceOf[ZincWorkerApi] } diff --git a/scalalib/test/resources/hello-java/core/src/Core.java b/scalalib/test/resources/hello-java/core/src/Core.java index 3ecb1f61032..313d7711eee 100644 --- a/scalalib/test/resources/hello-java/core/src/Core.java +++ b/scalalib/test/resources/hello-java/core/src/Core.java @@ -1,7 +1,18 @@ package hello; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; + public class Core{ public static String msg(){ return "Hello World"; } -} \ No newline at end of file + + public static void main(String[] args) throws IOException { + Path path = Paths.get(args[0]); + String version = System.getProperty("java.version"); + Files.write(path, version.getBytes()); + } +} diff --git a/scalalib/test/resources/hello-java/core/test/src/MyCoreTests.java b/scalalib/test/resources/hello-java/core/test/src/MyCoreTests.java index 38bebaebe46..e11c78c8fd1 100644 --- a/scalalib/test/resources/hello-java/core/test/src/MyCoreTests.java +++ b/scalalib/test/resources/hello-java/core/test/src/MyCoreTests.java @@ -1,6 +1,7 @@ package hello; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import org.junit.Test; public class MyCoreTests { @@ -12,4 +13,22 @@ public void msgTest() { public void lengthTest() { assertEquals(Core.msg().length(), 11); } -} \ No newline at end of file + + @Test + public void java11Test() { + String version = System.getProperty("java.version"); + int dot = version.indexOf("."); + assertNotEquals(dot, -1); + System.out.println(version); + assertEquals(version.substring(0, dot), "11"); + } + + @Test + public void java17Test() { + String version = System.getProperty("java.version"); + int dot = version.indexOf("."); + assertNotEquals(dot, -1); + System.out.println(version); + assertEquals(version.substring(0, dot), "17"); + } +} diff --git a/scalalib/test/resources/java-scala-11/javamodule/src/Main.java b/scalalib/test/resources/java-scala-11/javamodule/src/Main.java new file mode 100644 index 00000000000..5ecf9991507 --- /dev/null +++ b/scalalib/test/resources/java-scala-11/javamodule/src/Main.java @@ -0,0 +1,14 @@ +package hello; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; + +public class Main{ + public static void main(String[] args) throws IOException { + Path path = Paths.get(args[0]); + String version = System.getProperty("java.version"); + Files.writeString(path, version.indent(2)); + } +} diff --git a/scalalib/test/resources/java-scala-11/scalamodule/src/Main.scala b/scalalib/test/resources/java-scala-11/scalamodule/src/Main.scala new file mode 100644 index 00000000000..c37b3135df0 --- /dev/null +++ b/scalalib/test/resources/java-scala-11/scalamodule/src/Main.scala @@ -0,0 +1,14 @@ +package hello + +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.Path + +object Main { + def main(args: Array[String]): Unit = { + val path = Paths.get(args(0)) + val version = System.getProperty("java.version") + Files.writeString(path, version.indent(2)) + } +} diff --git a/scalalib/test/src/mill/scalalib/HelloJavaTests.scala b/scalalib/test/src/mill/scalalib/HelloJavaTests.scala index 7c8a0461087..f4298dd3387 100644 --- a/scalalib/test/src/mill/scalalib/HelloJavaTests.scala +++ b/scalalib/test/src/mill/scalalib/HelloJavaTests.scala @@ -5,7 +5,7 @@ import mill.api.Result import mill.testkit.UnitTester import mill.testkit.TestBaseModule import utest._ -import utest.framework.TestPath +import mill.define.ModuleRef object HelloJavaTests extends TestSuite { @@ -25,6 +25,7 @@ object HelloJavaTests extends TestSuite { } } } + val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-java" def testEval() = UnitTester(HelloJava, resourcePath) @@ -47,6 +48,7 @@ object HelloJavaTests extends TestSuite { !os.walk(result3.value.classes.path).exists(_.last == "Core.class") ) } + test("semanticDbData") { val expectedFile1 = os.rel / "META-INF/semanticdb/core/src/Core.java.semanticdb" @@ -152,10 +154,12 @@ object HelloJavaTests extends TestSuite { val Left(Result.Failure(ref1, Some(v1))) = eval.apply(HelloJava.core.test.test()) assert( - v1._2(0).fullyQualifiedName == "hello.MyCoreTests.lengthTest", - v1._2(0).status == "Success", - v1._2(1).fullyQualifiedName == "hello.MyCoreTests.msgTest", - v1._2(1).status == "Failure" + v1._2(0).fullyQualifiedName == "hello.MyCoreTests.java11Test", + v1._2(1).fullyQualifiedName == "hello.MyCoreTests.java17Test", + v1._2(2).fullyQualifiedName == "hello.MyCoreTests.lengthTest", + v1._2(2).status == "Success", + v1._2(3).fullyQualifiedName == "hello.MyCoreTests.msgTest", + v1._2(3).status == "Failure" ) val Right(result2) = eval.apply(HelloJava.app.test.test()) diff --git a/scalalib/test/src/mill/scalalib/JavaHomeTests.scala b/scalalib/test/src/mill/scalalib/JavaHomeTests.scala new file mode 100644 index 00000000000..a0c9384fcf6 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/JavaHomeTests.scala @@ -0,0 +1,181 @@ +package mill.scalalib + +import mill.define.Task +import mill.define.Args +import mill.api.Result +import mill.define.ModuleRef +import mill.testkit.{TestBaseModule, UnitTester} +import utest.* + +import java.io.{ByteArrayOutputStream, PrintStream} + +object JavaHomeTests extends TestSuite { + + object HelloJavaJavaHome11Override extends TestBaseModule { + object ZincWorkerJava11 extends ZincWorkerModule { + def jvmId = "temurin:11.0.24" + } + + object core extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava11) + override def docJarUseArgsFile = false + object test extends JavaTests with TestModule.Junit4 + } + } + + object HelloJavaJavaHome17Override extends TestBaseModule { + object ZincWorkerJava17 extends ZincWorkerModule { + def jvmId = "temurin:17.0.9" + } + + object core extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava17) + override def docJarUseArgsFile = false + object test extends JavaTests with TestModule.Junit4 + } + } + + object JavaJdk11DoesntCompile extends TestBaseModule { + object ZincWorkerJava extends ZincWorkerModule { + def jvmId = "temurin:11.0.25" + } + + object javamodule extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava) + } + object scalamodule extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava) + def scalaVersion = "2.13.14" + } + } + + object JavaJdk17Compiles extends TestBaseModule { + object ZincWorkerJava extends ZincWorkerModule { + def jvmId = "temurin:17.0.13" + } + + object javamodule extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava) + } + object scalamodule extends JavaModule { + override def zincWorker = ModuleRef(ZincWorkerJava) + + def scalaVersion = "2.13.14" + } + } + + val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-java" + + def tests: Tests = Tests { + + test("javaHome") { + test("compile11") { + val eval = UnitTester(HelloJavaJavaHome11Override, resourcePath) + + val Right(result) = eval.apply(HelloJavaJavaHome11Override.core.compile) + + val coreClassFile = os.walk(result.value.classes.path).find(_.last == "Core.class") + + assert( + coreClassFile.isDefined, + + // The first eight bytes are magic numbers followed by two bytes for major version and two bytes for minor version + // We are overriding to java 11 which corresponds to class file version 55 + os.read.bytes(coreClassFile.get, 4, 4).toSeq == Seq[Byte](0, 0, 0, 55) + ) + } + test("compile17") { + val eval = UnitTester(HelloJavaJavaHome17Override, resourcePath) + + val Right(result) = eval.apply(HelloJavaJavaHome17Override.core.compile) + + val coreClassFile = os.walk(result.value.classes.path).find(_.last == "Core.class") + + assert( + coreClassFile.isDefined, + + // The first eight bytes are magic numbers followed by two bytes for major version and two bytes for minor version + // We are overriding to java 17 which corresponds to class file version 67 + os.read.bytes(coreClassFile.get, 4, 4).toSeq == Seq[Byte](0, 0, 0, 61) + ) + } + test("run11") { + val eval = UnitTester(HelloJavaJavaHome11Override, resourcePath) + + val path = eval.evaluator.workspace / "java.version" + val Right(_) = eval.apply(HelloJavaJavaHome11Override.core.run(Task.Anon(Args(path)))) + + assert( + os.read(path).startsWith("11.") + ) + } + test("run17") { + val eval = UnitTester(HelloJavaJavaHome17Override, resourcePath) + + val path = eval.evaluator.workspace / "java.version" + val Right(_) = eval.apply(HelloJavaJavaHome17Override.core.run(Task.Anon(Args(path)))) + + assert( + os.read(path).startsWith("17.") + ) + } + test("test11") { + val eval = UnitTester(HelloJavaJavaHome11Override, resourcePath) + + val Left(Result.Failure(ref1, Some(v1))) = + eval.apply(HelloJavaJavaHome11Override.core.test.test()) + + assert( + v1._2(0).fullyQualifiedName == "hello.MyCoreTests.java11Test", + v1._2(0).status == "Success", + v1._2(1).fullyQualifiedName == "hello.MyCoreTests.java17Test", + v1._2(1).status == "Failure" + ) + } + test("test17") { + val eval = UnitTester(HelloJavaJavaHome17Override, resourcePath) + + val Left(Result.Failure(ref1, Some(v1))) = + eval.apply(HelloJavaJavaHome17Override.core.test.test()) + + assert( + v1._2(0).fullyQualifiedName == "hello.MyCoreTests.java11Test", + v1._2(0).status == "Failure", + v1._2(1).fullyQualifiedName == "hello.MyCoreTests.java17Test", + v1._2(1).status == "Success" + ) + } + } + + test("compileApis") { + val resourcePathCompile = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "java-scala-11" + test("jdk11java") { + val baos = new ByteArrayOutputStream() + val eval = + UnitTester(JavaJdk11DoesntCompile, resourcePathCompile, errStream = new PrintStream(baos)) + val Left(result) = eval.apply(JavaJdk11DoesntCompile.javamodule.compile) + assert(baos.toString.contains("cannot find symbol")) + assert(baos.toString.contains("method indent")) + } + + test("jdk17java") { + val eval = UnitTester(JavaJdk17Compiles, resourcePathCompile) + val Right(result) = eval.apply(JavaJdk17Compiles.javamodule.compile) + } + + // This doesn't work because Zinc doesn't apply the javaHome config to + // the Scala compiler JVM, which always runs in-memory https://github.com/sbt/zinc/discussions/1498 +// test("jdk11scala"){ +// val baos = new ByteArrayOutputStream() +// val eval = UnitTester(JavaJdk11DoesntCompile, resourcePathCompile) +// val Left(result) = eval.apply(JavaJdk11DoesntCompile.scalamodule.compile) +// } + + test("jdk17scala") { + val eval = UnitTester(JavaJdk17Compiles, resourcePathCompile) + val Right(result) = eval.apply(JavaJdk17Compiles.scalamodule.compile) + } + } + + } +} diff --git a/scalalib/worker/src/mill/scalalib/worker/DiscoverMainClassesMain.scala b/scalalib/worker/src/mill/scalalib/worker/DiscoverMainClassesMain.scala new file mode 100644 index 00000000000..2ba72ffdc6b --- /dev/null +++ b/scalalib/worker/src/mill/scalalib/worker/DiscoverMainClassesMain.scala @@ -0,0 +1,84 @@ +package mill.scalalib.worker + +import java.io.{File, PrintWriter} +import mill.api.{internal, DummyOutputStream} +import scala.annotation.tailrec +import scala.tools.nsc.{CloseableRegistry, Settings} +import scala.tools.nsc.classpath.{AggregateClassPath, ClassPathFactory} +import scala.tools.scalap.{ByteArrayReader, Classfile, JavaWriter} +import scala.util.Using + +@internal object DiscoverMainClassesMain { + // copied from ModuleUtils + private def recursive[T <: String](start: T, deps: T => Seq[T]): Seq[T] = { + + @tailrec def rec( + seenModules: List[T], + toAnalyze: List[(List[T], List[T])] + ): List[T] = { + toAnalyze match { + case Nil => seenModules + case traces :: rest => + traces match { + case (_, Nil) => rec(seenModules, rest) + case (trace, cand :: remaining) => + if (trace.contains(cand)) { + // cycle! + val rendered = + (cand :: (cand :: trace.takeWhile(_ != cand)).reverse).mkString(" -> ") + val msg = s"cycle detected: ${rendered}" + println(msg) + throw sys.error(msg) + } + rec( + if (seenModules.contains(cand)) seenModules + else { seenModules ++ Seq(cand) }, + toAnalyze = ((cand :: trace, deps(cand).toList)) :: (trace, remaining) :: rest + ) + } + } + } + + rec( + seenModules = List(), + toAnalyze = List((List(start), deps(start).toList)) + ).reverse + } + + def main(args: Array[String]): Unit = { + val classpath = args(0).split(",").map(os.Path(_)).toSeq + apply(classpath).foreach(println) + } + + def apply(classpath: Seq[os.Path]): Seq[String] = { + val cp = classpath.map(_.toNIO.toString()).mkString(File.pathSeparator) + + val settings = new Settings() + Using.resource(new CloseableRegistry) { registry => + val path = AggregateClassPath( + new ClassPathFactory(settings, registry).classesInExpandedPath(cp) + ) + + val mainClasses = for { + foundPackage <- recursive("", (p: String) => path.packages(p).map(_.name)) + classFile <- path.classes(foundPackage) + path = os.Path(classFile.file.file) + if path.ext == "class" + cf = { + val bytes = os.read.bytes(path) + val reader = new ByteArrayReader(bytes) + new Classfile(reader) + } + jw = new JavaWriter(cf, new PrintWriter(DummyOutputStream)) + method <- cf.methods + static = jw.isStatic(method.flags) + methodName = jw.getName(method.name) + methodType = jw.getType(method.tpe) + if static && methodName == "main" && methodType == "(scala.Array[java.lang.String]): scala.Unit" + className = jw.getClassName(cf.classname) + } yield className + + mainClasses + } + } +} diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index bcc0d47a027..5c15ede537c 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -1,14 +1,7 @@ package mill.scalalib.worker import mill.api.Loose.Agg -import mill.api.{ - CompileProblemReporter, - DummyOutputStream, - KeyedLockedCache, - PathRef, - Result, - internal -} +import mill.api.{CompileProblemReporter, KeyedLockedCache, PathRef, Result, internal} import mill.scalalib.api.{CompilationResult, Versions, ZincWorkerApi, ZincWorkerUtil} import sbt.internal.inc.{ Analysis, @@ -41,16 +34,11 @@ import xsbti.compile.{ } import xsbti.{PathBasedFile, VirtualFile} -import java.io.{File, PrintWriter} +import java.io.File import java.util.Optional -import scala.annotation.tailrec import scala.collection.mutable import scala.ref.SoftReference -import scala.tools.nsc.{CloseableRegistry, Settings} -import scala.tools.nsc.classpath.{AggregateClassPath, ClassPathFactory} -import scala.tools.scalap.{ByteArrayReader, Classfile, JavaWriter} import scala.util.Properties.isWin -import scala.util.Using @internal class ZincWorkerImpl( @@ -62,7 +50,8 @@ class ZincWorkerImpl( compilerJarNameGrep: (Agg[PathRef], String) => PathRef, compilerCache: KeyedLockedCache[Compilers], compileToJar: Boolean, - zincLogDebug: Boolean + zincLogDebug: Boolean, + javaHome: Option[PathRef] ) extends ZincWorkerApi with AutoCloseable { private val zincLogLevel = if (zincLogDebug) sbt.util.Level.Debug else sbt.util.Level.Info private[this] val ic = new sbt.internal.inc.IncrementalCompilerImpl() @@ -113,12 +102,13 @@ class ZincWorkerImpl( } private def getLocalOrCreateJavaTools(javacRuntimeOptions: Seq[String]): JavaTools = { + val javaHome = this.javaHome.map(_.path.toNIO) val (javaCompiler, javaDoc) = // Local java compilers don't accept -J flags so when we put this together if we detect // any javacOptions starting with -J we ensure we have a non-local Java compiler which // can handle them. - if (javacRuntimeOptions.exists(filterJavacRuntimeOptions)) { - (javac.JavaCompiler.fork(None), javac.Javadoc.fork(None)) + if (javacRuntimeOptions.exists(filterJavacRuntimeOptions) || javaHome.isDefined) { + (javac.JavaCompiler.fork(javaHome), javac.Javadoc.fork(javaHome)) } else { val compiler = javac.JavaCompiler.local.getOrElse(javac.JavaCompiler.fork(None)) @@ -301,33 +291,7 @@ class ZincWorkerImpl( * This implementation is only in this "zinc"-specific module, because this module is already shared between all `JavaModule`s. */ override def discoverMainClasses(classpath: Seq[os.Path]): Seq[String] = { - val cp = classpath.map(_.toNIO.toString()).mkString(File.pathSeparator) - - val settings = new Settings() - Using.resource(new CloseableRegistry) { registry => - val path = AggregateClassPath( - new ClassPathFactory(settings, registry).classesInExpandedPath(cp) - ) - - val mainClasses = for { - foundPackage <- ZincWorkerImpl.recursive("", (p: String) => path.packages(p).map(_.name)) - classFile <- path.classes(foundPackage) - cf = { - val bytes = os.read.bytes(os.Path(classFile.file.file)) - val reader = new ByteArrayReader(bytes) - new Classfile(reader) - } - jw = new JavaWriter(cf, new PrintWriter(DummyOutputStream)) - method <- cf.methods - static = jw.isStatic(method.flags) - methodName = jw.getName(method.name) - methodType = jw.getType(method.tpe) - if static && methodName == "main" && methodType == "(scala.Array[java.lang.String]): scala.Unit" - className = jw.getClassName(cf.classname) - } yield className - - mainClasses - } + DiscoverMainClassesMain(classpath) } def discoverMainClasses(compilationResult: CompilationResult): Seq[String] = { @@ -672,41 +636,3 @@ class ZincWorkerImpl( javaOnlyCompilersCache.clear() } } - -object ZincWorkerImpl { - // copied from ModuleUtils - private def recursive[T <: String](start: T, deps: T => Seq[T]): Seq[T] = { - - @tailrec def rec( - seenModules: List[T], - toAnalyze: List[(List[T], List[T])] - ): List[T] = { - toAnalyze match { - case Nil => seenModules - case traces :: rest => - traces match { - case (_, Nil) => rec(seenModules, rest) - case (trace, cand :: remaining) => - if (trace.contains(cand)) { - // cycle! - val rendered = - (cand :: (cand :: trace.takeWhile(_ != cand)).reverse).mkString(" -> ") - val msg = s"cycle detected: ${rendered}" - println(msg) - throw sys.error(msg) - } - rec( - if (seenModules.contains(cand)) seenModules - else { seenModules ++ Seq(cand) }, - toAnalyze = ((cand :: trace, deps(cand).toList)) :: (trace, remaining) :: rest - ) - } - } - } - - rec( - seenModules = List(), - toAnalyze = List((List(start), deps(start).toList)) - ).reverse - } -} diff --git a/testrunner/src/mill/testrunner/DiscoverTestsMain.scala b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala new file mode 100644 index 00000000000..f9bdb77c794 --- /dev/null +++ b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala @@ -0,0 +1,36 @@ +package mill.testrunner + +import mill.api.{Ctx, internal} +import os.Path + +@internal object DiscoverTestsMain { + private implicit def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] = + mill.api.JsonFormatters.PathTokensReader2 + + @mainargs.main + def main(runCp: Seq[os.Path], testCp: Seq[os.Path], framework: String): Unit = { + main0(runCp, testCp, framework).foreach(println) + } + def main0(runCp: Seq[os.Path], testCp: Seq[os.Path], framework: String): Seq[String] = { + mill.util.Jvm.inprocess( + runCp, + classLoaderOverrideSbtTesting = true, + isolated = true, + closeContextClassLoaderWhenDone = false, + body = classLoader => { + TestRunnerUtils + .discoverTests(classLoader, Framework.framework(framework)(classLoader), testCp) + .toSeq + .map(_._1.getName()) + .map { + case s if s.endsWith("$") => s.dropRight(1) + case s => s + } + } + )(new Ctx.Home { + def home: Path = os.home + }) + } + + def main(args: Array[String]): Unit = mainargs.ParserForMethods(this).runOrExit(args) +} diff --git a/testrunner/src/mill/testrunner/GetTestTasksMain.scala b/testrunner/src/mill/testrunner/GetTestTasksMain.scala new file mode 100644 index 00000000000..a3f63575d77 --- /dev/null +++ b/testrunner/src/mill/testrunner/GetTestTasksMain.scala @@ -0,0 +1,49 @@ +package mill.testrunner + +import mill.api.Loose.Agg +import mill.api.{Ctx, internal} +import os.Path + +@internal object GetTestTasksMain { + private implicit def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] = + mill.api.JsonFormatters.PathTokensReader2 + @mainargs.main + def main( + runCp: Seq[os.Path], + testCp: Seq[os.Path], + framework: String, + selectors: Seq[String], + args: Seq[String] + ): Unit = { + main0(runCp, testCp, framework, selectors, args).foreach(println) + } + + def main0( + runCp: Seq[os.Path], + testCp: Seq[os.Path], + framework: String, + selectors: Seq[String], + args: Seq[String] + ): Seq[String] = { + val globFilter = TestRunnerUtils.globFilter(selectors) + mill.util.Jvm.inprocess( + runCp, + classLoaderOverrideSbtTesting = true, + isolated = true, + closeContextClassLoaderWhenDone = false, + classLoader => + TestRunnerUtils + .getTestTasks0( + Framework.framework(framework), + Agg.from(testCp), + args, + cls => globFilter(cls.getName), + classLoader + ) + )(new Ctx.Home { + def home: Path = os.home + }) + } + + def main(args: Array[String]): Unit = mainargs.ParserForMethods(this).runOrExit(args) +} From 1fdabeec4e379597be5bb32158c929e79eefb6b2 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 20 Nov 2024 12:19:40 +0800 Subject: [PATCH 4/6] Add a new doc page `Why Mill? / How Fast Does Java Compile?` (#3990) --- docs/modules/ROOT/nav.adoc | 2 +- .../pages/cli/alternate-installation.adoc | 120 ------ .../ROOT/pages/cli/installation-ide.adoc | 121 ++++++ .../ROOT/pages/comparisons/java-compile.adoc | 343 ++++++++++++++++++ .../modules/ROOT/pages/comparisons/maven.adoc | 95 +++-- .../ROOT/pages/comparisons/why-mill.adoc | 12 +- 6 files changed, 517 insertions(+), 176 deletions(-) delete mode 100644 docs/modules/ROOT/pages/cli/alternate-installation.adoc create mode 100644 docs/modules/ROOT/pages/comparisons/java-compile.adoc diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 37fe0701af1..919e933756e 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -37,12 +37,12 @@ ** xref:comparisons/gradle.adoc[] ** xref:comparisons/sbt.adoc[] ** xref:comparisons/unique.adoc[] +** xref:comparisons/java-compile.adoc[] * The Mill CLI ** xref:cli/installation-ide.adoc[] ** xref:cli/flags.adoc[] ** xref:cli/builtin-commands.adoc[] ** xref:cli/query-syntax.adoc[] -** xref:cli/alternate-installation.adoc[] * Migrating to Mill ** xref:migrating/maven.adoc[] // This section gives a tour of the various user-facing features of Mill: diff --git a/docs/modules/ROOT/pages/cli/alternate-installation.adoc b/docs/modules/ROOT/pages/cli/alternate-installation.adoc deleted file mode 100644 index 0f46fc9e839..00000000000 --- a/docs/modules/ROOT/pages/cli/alternate-installation.adoc +++ /dev/null @@ -1,120 +0,0 @@ -= Other installation methods - -CAUTION: The installation methods listed below are maintained outside of Mill and may not have -the same features as the xref:cli/installation-ide.adoc#_bootstrap_scripts[bootstrap scripts]. You can try using them, -but the officially supported way to use Mill is via the bootstrap script above, so the Mill -maintainers may be unable to help you if you have issues with some alternate installation method. - -CAUTION: Some of the installations via package managers install a fixed version of Mill and -do not support project-specific selection of the preferred Mill version. If you want to use -the `MILL_VERSION` environment variable or need support for `.mill-version` or -`.config/mill-version` files to control the actual used Mill version, please use -a xref:cli/installation-ide.adoc#_bootstrap_scripts[bootstrap script] instead. - -== OS X - -Installation via https://github.com/Homebrew/homebrew-core/blob/master/Formula/m/mill.rb[homebrew]: - -[source,sh] ----- -brew install mill ----- - - -== Arch Linux - -Arch Linux has an https://archlinux.org/packages/extra/any/mill/[Extra package for mill]: - -[source,bash] ----- -pacman -S mill - ----- - -== FreeBSD - -Installation via http://man.freebsd.org/pkg/8[pkg(8)]: - -[source,sh] ----- -pkg install mill - ----- - -== Gentoo Linux - -[source,sh] ----- -emerge dev-java/mill-bin - ----- - -== Windows - -To get started, download Mill from -{mill-github-url}/releases/download/{mill-last-tag}/{mill-last-tag}-assembly[Github releases], and save it as `mill.bat`. - -If you're using https://scoop.sh[Scoop] you can install Mill via - -[source,bash] ----- -scoop install mill ----- - -== WSL / MSYS2 / Cycgin / Git-Bash - -Mill also works on "sh" environments on Windows (e.g., -https://www.msys2.org[MSYS2], -https://www.cygwin.com[Cygwin], -https://gitforwindows.org[Git-Bash], -https://docs.microsoft.com/en-us/windows/wsl[WSL]); to get started, follow the instructions in the <<_manual>> -section. Note that: - -* In some environments (such as WSL), Mill might have to be run without a server (using `-i`, `--interactive`, or `--no-server`.) - -* On Cygwin, run the following after downloading mill: - -[source,bash] ----- -sed -i '0,/-cp "\$0"/{s/-cp "\$0"/-cp `cygpath -w "\$0"`/}; 0,/-cp "\$0"/{s/-cp "\$0"/-cp `cygpath -w "\$0"`/}' /usr/local/bin/mill ----- - -== Docker - -You can download and run -a https://hub.docker.com/r/nightscape/scala-mill/["Docker image containing OpenJDK, Scala and Mill"] using - -[source,bash] ----- -docker pull nightscape/scala-mill -docker run -it nightscape/scala-mill ----- - -== Manual - -To get started, download Mill and install it into your HOME ".local/bin" via the following -`curl`/`chmod` command: - -[source,bash,subs="verbatim,attributes"] ----- -sh -c "curl -L {mill-github-url}/releases/download/{mill-last-tag}/{mill-last-tag} > ~/.local/bin/mill && chmod +x ~/.local/bin/mill" ----- - -== Coursier (unsupported) - -Installing mill via `coursier` or `cs` is currently not officially supported. -There are various issues, especially with interactive mode. - -== Asdf (unsupported) - -You can install and manage Mill via the Multiple Runtime Version Manager - https://asdf-vm.com/[`asdf`]. - -Support by `asdf` is currently possible by using the https://github.com/asdf-community/asdf-mill[`asdf-mill` plugin]: - -.Steps to install the `mill` plugin and Mill with `asdf` -[source,bash] ---- -asdf plugin add mill -asdf install mill latest -asdf global mill latest ---- diff --git a/docs/modules/ROOT/pages/cli/installation-ide.adoc b/docs/modules/ROOT/pages/cli/installation-ide.adoc index fa8c42b3ccf..f2c271c0e61 100644 --- a/docs/modules/ROOT/pages/cli/installation-ide.adoc +++ b/docs/modules/ROOT/pages/cli/installation-ide.adoc @@ -256,3 +256,124 @@ The easiest way to use a development release is to use one of the `MILL_VERSION` environment variable or a `.mill-version` or `.config/mill-version` file. + +== Other installation methods + +CAUTION: The installation methods listed below are maintained outside of Mill and may not have +the same features as the xref:cli/installation-ide.adoc#_bootstrap_scripts[bootstrap scripts]. You can try using them, +but the officially supported way to use Mill is via the bootstrap script above, so the Mill +maintainers may be unable to help you if you have issues with some alternate installation method. + +CAUTION: Some of the installations via package managers install a fixed version of Mill and +do not support project-specific selection of the preferred Mill version. If you want to use +the `MILL_VERSION` environment variable or need support for `.mill-version` or +`.config/mill-version` files to control the actual used Mill version, please use +a xref:cli/installation-ide.adoc#_bootstrap_scripts[bootstrap script] instead. + +=== OS X + +Installation via https://github.com/Homebrew/homebrew-core/blob/master/Formula/m/mill.rb[homebrew]: + +[source,sh] +---- +brew install mill +---- + + +=== Arch Linux + +Arch Linux has an https://archlinux.org/packages/extra/any/mill/[Extra package for mill]: + +[source,bash] +---- +pacman -S mill + +---- + +=== FreeBSD + +Installation via http://man.freebsd.org/pkg/8[pkg(8)]: + +[source,sh] +---- +pkg install mill + +---- + +=== Gentoo Linux + +[source,sh] +---- +emerge dev-java/mill-bin + +---- + +=== Windows + +To get started, download Mill from +{mill-github-url}/releases/download/{mill-last-tag}/{mill-last-tag}-assembly[Github releases], and save it as `mill.bat`. + +If you're using https://scoop.sh[Scoop] you can install Mill via + +[source,bash] +---- +scoop install mill +---- + +=== WSL / MSYS2 / Cycgin / Git-Bash + +Mill also works on "sh" environments on Windows (e.g., +https://www.msys2.org[MSYS2], +https://www.cygwin.com[Cygwin], +https://gitforwindows.org[Git-Bash], +https://docs.microsoft.com/en-us/windows/wsl[WSL]); to get started, follow the instructions in the <<_manual>> +section. Note that: + +* In some environments (such as WSL), Mill might have to be run without a server (using `-i`, `--interactive`, or `--no-server`.) + +* On Cygwin, run the following after downloading mill: + +[source,bash] +---- +sed -i '0,/-cp "\$0"/{s/-cp "\$0"/-cp `cygpath -w "\$0"`/}; 0,/-cp "\$0"/{s/-cp "\$0"/-cp `cygpath -w "\$0"`/}' /usr/local/bin/mill +---- + +=== Docker + +You can download and run +a https://hub.docker.com/r/nightscape/scala-mill/["Docker image containing OpenJDK, Scala and Mill"] using + +[source,bash] +---- +docker pull nightscape/scala-mill +docker run -it nightscape/scala-mill +---- + +=== Manual + +To get started, download Mill and install it into your HOME ".local/bin" via the following +`curl`/`chmod` command: + +[source,bash,subs="verbatim,attributes"] +---- +sh -c "curl -L {mill-github-url}/releases/download/{mill-last-tag}/{mill-last-tag} > ~/.local/bin/mill && chmod +x ~/.local/bin/mill" +---- + +=== Coursier (unsupported) + +Installing mill via `coursier` or `cs` is currently not officially supported. +There are various issues, especially with interactive mode. + +=== Asdf (unsupported) + +You can install and manage Mill via the Multiple Runtime Version Manager - https://asdf-vm.com/[`asdf`]. + +Support by `asdf` is currently possible by using the https://github.com/asdf-community/asdf-mill[`asdf-mill` plugin]: + +.Steps to install the `mill` plugin and Mill with `asdf` +[source,bash] +--- +asdf plugin add mill +asdf install mill latest +asdf global mill latest +--- diff --git a/docs/modules/ROOT/pages/comparisons/java-compile.adoc b/docs/modules/ROOT/pages/comparisons/java-compile.adoc new file mode 100644 index 00000000000..63d9ec3fed8 --- /dev/null +++ b/docs/modules/ROOT/pages/comparisons/java-compile.adoc @@ -0,0 +1,343 @@ +# How Fast Does Java Compile? + +include::partial$gtag-config.adoc[] + +Java compiles have the reputation for being slow, but that reputation does +not match today's reality. Nowadays the Java compiler can compile "typical" Java code at over +100,000 lines a second on a single core. That means that even a million line project +should take more than 10s to compile in a single-threaded fashion, and should be even +faster in the presence of parallelism + +Doing some ad-hoc benchmarks, we find that although the compiler is blazing fast, all +build tools add significant overhead over compiling Java directly: + +|=== +| Mockito Core | Time | Compiler lines/s | Multiplier | Netty Common | Time | Compiler lines/s | Multiplier +| Javac Hot | 0.36s | 115,600 | 1.0x | Javac Hot | 0.29s | 102,500 | 1.0x +| Javac Cold | 1.29s | 32,200 | 4.4x | Javac Cold | 1.62s | 18,300 | 5.6x +| Mill | 1.20s | 34,700 | 4.1x | Mill | 1.11s | 26,800 | 3.8x +| Gradle | 4.41s | 9,400 | 15.2x | Maven | 4.89s | 6,100 | 16.9x +|=== + +Although Mill does the best in these benchmarks among the build tools (Maven, Gradle, and Mill), +all build tools fall short of how fast compiling Java _should_ be. This post explores how +these numbers were arrived at, and what that means in un-tapped potential for Java build +tooling to become truly great. + +## Mockito Core + +To begin to understand the problem, lets consider the codebase of the popular Mockito project: + +* https://github.com/mockito/mockito + +Mockito is a medium-sized Java project with a few dozen sub-modules and about ~100,000 lines +of code. To give us a simple reproducible scenario, let's consider the root mockito module +with sources in `src/main/java/`, on which all the downstream module and tests depend on. + +Mockito is built using Gradle. It's not totally trivial to extract the compilation classpath +from Gradle, but the following stackoverflow answer gives us some tips: + +* https://stackoverflow.com/a/50639444/871202[How do I print out the Java classpath in gradle?] + +```bash +> ./gradlew clean && ./gradlew :classes --no-build-cache --debug | grep "classpath " +``` + +This gives us the following classpath: + +``` +export MY_CLASSPATH=/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/net.bytebuddy/byte-buddy/1.14.18/81e9b9a20944626e6757b5950676af901c2485/byte-buddy-1.14.18.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/net.bytebuddy/byte-buddy-agent/1.14.18/417558ea01fe9f0e8a94af28b9469d281c4e3984/byte-buddy-agent-1.14.18.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/junit/junit/4.13.2/8ac9e16d933b6fb43bc7f576336b8f4d7eb5ba12/junit-4.13.2.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/org.hamcrest/hamcrest-core/2.2/3f2bd07716a31c395e2837254f37f21f0f0ab24b/hamcrest-core-2.2.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/org.opentest4j/opentest4j/1.3.0/152ea56b3a72f655d4fd677fc0ef2596c3dd5e6e/opentest4j-1.3.0.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/org.objenesis/objenesis/3.3/1049c09f1de4331e8193e579448d0916d75b7631/objenesis-3.3.jar:/Users/lihaoyi/.gradle/caches/modules-2/files-2.1/org.hamcrest/hamcrest/2.2/1820c0968dba3a11a1b30669bb1f01978a91dedc/hamcrest-2.2.jar +``` + +Note that for this benchmark, all third-party dependencies have already been resolved +and downloaded from Maven Central. We can thus simply reference the jars on disk directly, +which we do above. + +We can then pass this classpath into `javac -cp`, together with `src/main/java/**/*.java`, +to perform the compilation outside of Gradle using `javac` directly. Running this a few +times gives us the timings below: + +```bash +> time javac -cp $MY_CLASSPATH src/main/java/**/*.java +1.290s +1.250s +1.293s +``` + +To give us an idea of how many lines of code we are compiling, we can run: + +```bash +> find src/main/java | grep \\.java | xargs wc -l +... +41601 total +``` + +Combining this information, we find that 41601 lines of code compiled in ~1.29 seconds +(taking the median of the three runs above) suggests that `javac` compiles about ~32,000 +lines of code per second. + +These benchmarks were run ad-hoc on my laptop, an M1 10-core Macbook Pro, with OpenJDK +Corretto 17.0.6. The numbers would differ on different Java versions, hardware, operating systems, +and filesystems. Nevertheless, the overall trend is strong enough that you should be +able to reproduce the results despite variations in the benchmarking environment. + +Compiling 32,000 lines of code per second is not bad. But it is nowhere near how fast the +Java compiler _can_ run. Any software experience with JVM experience would know the next +obvious optimization for us to explore. + +## Keeping the JVM Hot + +One issue with the above benchmark is that it uses `javac` as a sub-process. The Java +compiler runs on the Java Virtual Machine, and like any JVM application, it has a slow +startup time, takes time warming-up, but then has good steady-state performance. +Running `javac` from the command line and compiling ~32,000 lines/sec is thus the _worst_ +possible performance you could get out of the Java compiler on this Java codebase. + +To get good performance out of `javac`, like any other JVM application, we need to keep it +long-lived so it has a chance to warm up. While running the `javac` in a long-lived Java +program is not commonly taught, neither is it particularly difficult. Here is a complete +`Bench.java` file that does this, repeatedly running java compilation in a loop where it +has a chance to warm up, to emulate the long lived JVM process that a build tool like Mill +may spawn and manage. We use the same `MY_CLASSPATH` and source files we saw earlier and +print the output statistics to the terminal so we can see how fast Java compilation can +occur once things have a chance to warm up: + +```java +// Bench.java +import javax.tools.*; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.file.*; +import java.util.List; +import java.util.stream.Collectors; + +public class Bench { + public static void main(String[] args) throws Exception { + while (true) { + long now = System.currentTimeMillis(); + String classpath = System.getenv("MY_CLASSPATH"); + Path sourceFolder = Paths.get("src/main/java"); + + List files = Files.walk(sourceFolder) + .filter(p -> p.toString().endsWith(".java")) + .map(p -> + new SimpleJavaFileObject(p.toUri(), JavaFileObject.Kind.SOURCE) { + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return Files.readString(p); + } + } + ) + .collect(Collectors.toList()); + + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + StandardJavaFileManager fileManager = compiler + .getStandardFileManager(null, null, null); + + // Run the compiler + JavaCompiler.CompilationTask task = compiler.getTask( + new OutputStreamWriter(System.out), + fileManager, + null, + List.of("-classpath", classpath), + null, + files + ); + + System.out.println("Compile Result: " + task.call()); + long end = System.currentTimeMillis(); + long lineCount = Files.walk(sourceFolder) + .filter(p -> p.toString().endsWith(".java")) + .map(p -> { + try { return Files.readAllLines(p).size(); } + catch(Exception e){ throw new RuntimeException(e); } + }) + .reduce(0, (x, y) -> x + y); + System.out.println("Lines: " + lineCount); + System.out.println("Duration: " + (end - now)); + System.out.println("Lines/second: " + lineCount / ((end - now) / 1000)); + } + } +} +``` + +Running this using `java Bench.java` in the Mockito repo root, eventually we see it +settle on approximately the following numbers: + +```bash +359ms +378ms +353ms +``` + +The codebase hasn't changed - we are still compiling 41,601 lines of code - +but now it only takes ~359ms. That tells us that using a long-lived warm Java compiler +we can compile approximately *116,000* lines of Java a second on a single core. + + +Compiling 116,000 lines of Java per second is very fast. That means we should expect +a million-line Java codebase to compile in about 9 seconds, _on a single thread_. That +may seem surprisingly fast, and you may be forgiven if you find it hard to believe. As +mentioned earlier, this number is expected to vary based on the codebase being compiled; +could it be that Mockito-Core just happens to be a very simple Java module that compiles +quickly? + +## Double-checking Our Results + +To double-check our results, we can pick another codebase to run some ad-hoc benchmarks. +For this I will use the Netty codebase: + +- https://github.com/netty/netty + +Netty is a large-ish Java project: ~500,000 lines of code. Again, to pick a somewhat +easily-reproducible benchmark, we want a decently-sized module that's relatively +standalone within the project: `netty-common` is a perfect fit. Again, we can use `find | grep | xargs` +to see how many lines of code we are looking at: + +```bash +$ find common/src/main/java | grep \\.java | xargs wc -l +29712 total +``` + +Again, Maven doesn't make it easy to show the classpath used to call `javac` ourselves, +but the following stackoverflow answer gives us a hint in how to do so: + +- https://stackoverflow.com/a/16655088/871202[In Maven, how output the classpath being used?] + +```bash +> ./mvnw clean; time ./mvnw -e -X -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true install +``` + +If you grep the output for `-classpath`, we see: + +```bash +-classpath /Users/lihaoyi/Github/netty/common/target/classes:/Users/lihaoyi/.m2/repository/org/graalvm/nativeimage/svm/19.3.6/svm-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/sdk/graal-sdk/19.3.6/graal-sdk-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/nativeimage/objectfile/19.3.6/objectfile-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/nativeimage/pointsto/19.3.6/pointsto-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/truffle/truffle-nfi/19.3.6/truffle-nfi-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/truffle/truffle-api/19.3.6/truffle-api-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/graalvm/compiler/compiler/19.3.6/compiler-19.3.6.jar:/Users/lihaoyi/.m2/repository/org/jctools/jctools-core/4.0.5/jctools-core-4.0.5.jar:/Users/lihaoyi/.m2/repository/org/jetbrains/annotations-java5/23.0.0/annotations-java5-23.0.0.jar:/Users/lihaoyi/.m2/repository/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar:/Users/lihaoyi/.m2/repository/commons-logging/commons-logging/1.2/commons-logging-1.2.jar:/Users/lihaoyi/.m2/repository/org/apache/logging/log4j/log4j-1.2-api/2.17.2/log4j-1.2-api-2.17.2.jar:/Users/lihaoyi/.m2/repository/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.jar:/Users/lihaoyi/.m2/repository/io/projectreactor/tools/blockhound/1.0.6.RELEASE/blockhound-1.0.6.RELEASE.jar +``` + +Again, we can `export MY_CLASSPATH` and start using `javac` from the command line: + +```bash +> javac -cp $MY_CLASSPATH common/src/main/java/**/*.java +1.624s +1.757s +1.606s +``` + +Or programmatically using the `Bench.java` program we saw earlier: + +```bash +294ms +282ms +285ms +``` + +Taking 285ms for a hot-in-memory compile of 29,712 lines of code, `netty-common` +therefore compiles at *~104,000 lines/second*. + +Although the choice of project is arbitrary, Mockito-Core and Netty-Common are decent +examples of Java code found "out in the wild". They aren't synthetic fake codebases generated +for the purpose of benchmarks, nor are they particularly unusual or idiosyncratic. They follow +most Java best practices and adhere to many of the most common Java linters (although those +were disabled for this performance benchmark). This is Java code that looks just like +any Java code you may write in your own projects, and it effortlessless compiles at +>100,000 lines/second. + +## What About Build Tools? + +Although the Java Compiler is blazing fast - compiling code at >100k lines/second and +completing both Mockito-Core and Netty-Common in ~300ms - the experience of using typical Java +build tools is nowhere near as snappy. Consider the benchmark of clean-compiling the +Mockito-Core codebase using Gradle or Mill: + +```bash +$ ./gradlew clean; time ./gradlew :classes --no-build-cache +4.14s +4.41s +4.41s + +$ ./mill clean; time ./mill compile +1.20s +1.12s +1.30s +``` + +Or the benchmark of clean-compiling the Netty-Common codebase using Maven or Mill: + +```bash +$ ./mvnw clean; time ./mvnw -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true -Dmaven.test.skip=true install +4.85s +4.96s +4.89s + +$ ./mill clean common; time ./mill common.compile +1.10s +1.12s +1.11s +``` + +Tabulating this all together gives us the table we saw at the start of this page: + +|=== +| Mockito Core | Time | Compiler lines/s | Multiplier | Netty Common | Time | Compiler lines/s | Multiplier +| Javac Hot | 0.36s | 115,600 | 1.0x | Javac Hot | 0.29s | 102,500 | 1.0x +| Javac Cold | 1.29s | 32,200 | 4.4x | Javac Cold | 1.62s | 18,300 | 5.6x +| Mill | 1.20s | 34,700 | 4.1x | Mill | 1.11s | 26,800 | 3.8x +| Gradle | 4.41s | 9,400 | 15.2x | Maven | 4.89s | 6,100 | 16.9x +|=== + +We explore the comparison between xref:comparisons/gradle.adoc[Gradle vs Mill] +or xref:comparisons/maven.adoc[Maven vs Mill] in more detail on their own dedicated pages. +For this article, the important thing is not comparing the build tools against each other, +but comparing the build tools against what how fast they _could_ be if they just used +the `javac` Java compiler directly. And it's clear that compared to the actual work +done by `javac` to actually compile your code, build tools add a frankly absurd amount +of overhead ranging from ~4x for Mill to 15-16x for Maven and Gradle! + +## Whole Project Compile Speed + +One thing worth calling out is that the overhead of the various build tools does not +appear to go down in larger builds. This *Clean Compile Single-Module* benchmark +we explored above only deals with compiling a single small module. But a similar *Sequential +Clean Compile* benchmarks which compiles the entire Mockito and Netty projects on +a single core shows similar numbers for the various build tools: + +* xref:comparisons/gradle.adoc#_sequential_clean_compile_all[Gradle compiling 100,000 lines of Java at ~5,600 lines/s] +* xref:comparisons/maven.adoc#_sequential_clean_compile_all[Maven compiling 500,000 lines of Java at ~5,100 lines/s] +* Mill compiling at ~25,000 lines/s on both the above whole-project benchmarks + +All of these are far below the 100,000 lines/s that we should expect from Java compilation, +but they roughly line up with the numbers measured above. Again, these benchmarks are ad-hoc, +on arbitrary hardware and JVM versions. They do include small amounts of other work, such +as compiling C/C++ code in Netty or doing ad-hoc file operations in Mockito. However, +most of the time is still spent in compilation, and this reinforces the early finding +that build tools (especially older ones like Maven or Gradle) are indeed adding huge +amounts of overhead on top of the extremely-fast Java compiler. + +## Conclusion + +From this study we can see the paradox: the Java _compiler_ is blazing fast, +while Java _build tools_ are dreadfully slow. Something that _should_ compile in a fraction +of a second using a warm `javac` takes several seconds (15-16x longer) to +compile using Maven or Gradle. Mill does better, but even it adds 4x overhead and falls +short of the snappiness you would expect from a compiler that takes ~0.3s to compile the +30-40kLOC Java codebases we experimented with. + +These benchmarks were run ad-hoc and on my laptop on arbitrary codebases, and the details +will obviously differ depending on environment and the code in question. Running it on an +entire codebase, rather than a single module, will give different results. Nevertheless, the +results are clear: "typical" Java code _should_ compile at ~100,000 lines/second on a single +thread. Anything less is purely build-tool overhead from Maven, Gradle, or Mill. + +Build tools do a lot more than the Java compiler. They do dependency management, parallelism, +caching and invalidation, and all sorts of other auxiliary tasks. But in the common case where +someone edits code and then compiles it, and all your dependencies are already downloaded and +cached locally, any time doing other things and not spent _actually +compiling Java_ is pure overhead. Checking for cache invalidation in _shouldn't_ take 15-16x +as long as actually compiling your code. I mean it obviously does _today_, but it _shouldn't_! + +The Mill build tool goes to great lengths to try and minimize overhead, and already gets +xref:comparisons/why-mill.adoc#_performance[~4x faster builds] than Maven or Gradle on +real-world projects like Mockito or Netty. But there still is a long way to go give Java +developers the fast, snappy experience that the underlying Java platform can provide. If +Java build and compile times are things you find important, you should try out Mill on +your own projects and get involved in the effort! \ No newline at end of file diff --git a/docs/modules/ROOT/pages/comparisons/maven.adoc b/docs/modules/ROOT/pages/comparisons/maven.adoc index 51a28de54ec..c583a5c1b4e 100644 --- a/docs/modules/ROOT/pages/comparisons/maven.adoc +++ b/docs/modules/ROOT/pages/comparisons/maven.adoc @@ -73,11 +73,12 @@ Mill's performance compares to Maven: |=== | Benchmark | Maven | Mill | Speedup -| <> | 1m 38.80s | 23.14s | 4.3x -| <> | 0m 48.92s | 0m 08.79s | 5.6x -| <> | 0m 08.46s | 0m 01.94s | 4.4x -| <> | 0m 06.82s | 0m 00.54s | 12.6x -| <> | 0m 05.25s | 0m 00.47s | 11.2x + +| <> | 98.80s | 23.14s | 4.3x +| <> | 48.92s | 8.79s | 5.6x +| <> | 4.89s | 1.11s | 4.4x +| <> | 6.82s | 0.54s | 12.6x +| <> | 5.25s | 0.47s | 11.2x |=== The column on the right shows the speedups of how much faster Mill is compared to the @@ -89,14 +90,14 @@ we can explain the difference in performing the same task with the two different ```bash $ ./mvnw clean; time ./mvnw -Pfast -Dcheckstyle.skip -Denforcer.skip=true -DskipTests install -1m 38.80s -1m 36.14s -1m 39.95s +98.80s +96.14s +99.95s $ ./mill clean; time ./mill -j1 __.compile -0m 23.99s -0m 23.14s -0m 22.68s +23.99s +23.14s +22.68s ``` This benchmark exercises the simple "build everything from scratch" workflow, with all remote @@ -143,9 +144,9 @@ Maven's overhead adds to the clean build: ```bash $ ./mill clean; time ./mill -j1 __.jar -0m 32.58s -0m 24.90s -0m 23.35s +32.58s +24.90s +23.35s ``` From this benchmark, we can see that although both Mill and Maven are doing the same work, @@ -158,14 +159,14 @@ whereas Mill directly uses the classfiles generated on disk to bypass all that w ```bash $ ./mvnw clean; time ./mvnw -T 10 -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true install -0m 48.92s -0m 48.41s -0m 49.50s +48.92s +48.41s +49.50s $ ./mill clean; time ./mill __.compile -0m 09.07s -0m 08.79s -0m 07.93s +9.07s +8.79s +7.93s ``` This example compares Maven v.s. Mill, when performing the clean build on 10 threads. @@ -180,30 +181,26 @@ when performing a clean build of the Netty repository. === Clean Compile Single-Module ```bash -$ ./mvnw clean; time ./mvnw -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true install -0m 08.46s -0m 08.90s -0m 08.30s - -$ ./mill clean common; time ./mill common.test.compile -0m 01.99s -0m 01.81s -0m 01.94s +$ ./mvnw clean; time ./mvnw -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true -Dmaven.test.skip=true install +4.85s +4.96s +4.89s + +$ ./mill clean common; time ./mill common.compile +1.10s +1.12s +1.11s ``` -This exercise limits the comparison to compiling a single module, in this case `common/`. -`./mvnw -pl common install` compiles both the `main/` and `test/` sources, whereas -`./mill common.compile` would only compile the `main/` sources, and we need to explicitly -reference `common.test.compile` to compile both (because `common.test.compile` depends on -`common.compile`, `common.compile` gets run automatically) +This exercise limits the comparison to compiling a single module, in this case `common/`, +ignore test sources. Again, we can see a significant speedup of Mill v.s. Maven remains even when compiling a -single module: a clean compile of `common/` is about 9x faster with Mill than with Maven! -Again, `common/` is about 40,000 lines of Java source code, so at 10,000-50,000 lines per +single module: a clean compile of `common/` is about 4x faster with Mill than with Maven! +Again, `common/` is about 30,000 lines of Java source code, so at 10,000-50,000 lines per second we would expect it to compile in about 1-4s. That puts Mill's compile times right at what you would expect, whereas Maven's has a significant overhead. - === Incremental Compile Single-Module ```bash @@ -212,18 +209,18 @@ $ time ./mvnw -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=t Compiling 174 source files to /Users/lihaoyi/Github/netty/common/target/classes Compiling 60 source files to /Users/lihaoyi/Github/netty/common/target/test-classes -0m 06.89s -0m 06.34s -0m 06.82s +6.89s +6.34s +6.82s $ echo "" >> common/src/main/java/io/netty/util/AbstractConstant.java $ time ./mill common.test.compile compiling 1 Java source to /Users/lihaoyi/Github/netty/out/common/compile.dest/classes ... -0m 00.78s -0m 00.54s -0m 00.51s +0.78s +0.54s +0.51s ``` This benchmark explores editing a single file and re-compiling `common/`. @@ -308,14 +305,14 @@ the same thing in Maven ```bash $ time ./mvnw -pl common -Pfast -DskipTests -Dcheckstyle.skip -Denforcer.skip=true install -0m 05.08s -0m 05.25s -0m 05.26s +5.08s +5.25s +5.26s $ time ./mill common.test.compile -0m 00.49s -0m 00.47s -0m 00.45s +0.49s +0.47s +0.45s ``` This last benchmark explores the boundaries of Maven and Mill: what happens if diff --git a/docs/modules/ROOT/pages/comparisons/why-mill.adoc b/docs/modules/ROOT/pages/comparisons/why-mill.adoc index a4acbe052e9..2bd8eb7b044 100644 --- a/docs/modules/ROOT/pages/comparisons/why-mill.adoc +++ b/docs/modules/ROOT/pages/comparisons/why-mill.adoc @@ -62,11 +62,11 @@ both parallel and sequential, and for many modules or for a single module: |=== | Benchmark | Maven | Mill | Speedup -| xref:comparisons/maven.adoc#_sequential_clean_compile_all[Sequential Clean Compile All] | 1m 38.80s | 23.14s | 4.3x -| xref:comparisons/maven.adoc#_parallel_clean_compile_all[Parallel Clean Compile All] | 0m 48.92s | 0m 08.79s | 5.6x -| xref:comparisons/maven.adoc#_clean_compile_single_module[Clean Compile Single Module] | 0m 08.46s | 0m 01.94s | 4.4x -| xref:comparisons/maven.adoc#_incremental_compile_single_module[Incremental Compile Single Module] | 0m 06.82s | 0m 00.54s | 12.6x -| xref:comparisons/maven.adoc#_no_op_compile_single_module[No-Op Compile Single Module] | 0m 05.25s | 0m 00.47s | 11.2x +| xref:comparisons/maven.adoc#_sequential_clean_compile_all[Sequential Clean Compile All] | 98.80s | 23.14s | 4.3x +| xref:comparisons/maven.adoc#_parallel_clean_compile_all[Parallel Clean Compile All] | 48.92s | 8.79s | 5.6x +| xref:comparisons/maven.adoc#_clean_compile_single_module[Clean Compile Single Module] | 4.89s | 1.11s | 4.4x +| xref:comparisons/maven.adoc#_incremental_compile_single_module[Incremental Compile Single Module] | 6.82s | 0.54s | 12.6x +| xref:comparisons/maven.adoc#_no_op_compile_single_module[No-Op Compile Single Module] | 5.25s | 0.47s | 11.2x |=== First, let's look at *Parallel Clean Compile All*. @@ -93,7 +93,7 @@ across the various workflows: |=== | Benchmark | Gradle | Mill | Speedup -| xref:comparisons/maven.adoc#_sequential_clean_compile_all[Sequential Clean Compile All] | 17.6s | 5.40s | 3.3x +| xref:comparisons/maven.adoc#_sequential_clean_compile_all[Sequential Clean Compile All] | 1s | 5.40s | 3.3x | xref:comparisons/maven.adoc#_parallel_clean_compile_all[Parallel Clean Compile All] | 12.3s | 3.57s | 3.4x | xref:comparisons/maven.adoc#_clean_compile_single_module[Clean Compile Single Module] | 4.41s | 1.20s | 3.7x | xref:comparisons/maven.adoc#_incremental_compile_single_module[Incremental Compile Single Module] | 1.37s | 0.51s | 2.7x From adc78a56377baf2caf18cb50b935b71cc22c5cb1 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 20 Nov 2024 13:50:03 +0800 Subject: [PATCH 5/6] Update nav-tree.hbs --- docs/supplemental-ui/partials/nav-tree.hbs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/supplemental-ui/partials/nav-tree.hbs b/docs/supplemental-ui/partials/nav-tree.hbs index 53a8d6a810e..40fade0a109 100644 --- a/docs/supplemental-ui/partials/nav-tree.hbs +++ b/docs/supplemental-ui/partials/nav-tree.hbs @@ -12,7 +12,7 @@ show all of them all the time -{{/if}} \ No newline at end of file +{{/if}} From 422ef4094600af1cc4c44027a8357dcd5e37bb01 Mon Sep 17 00:00:00 2001 From: Myyk Seok <2080820+myyk@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:01:19 +0800 Subject: [PATCH 6/6] Docs nit 1 (#3996) Fixes: https://mill-build.org/mill/extending/example-typescript-support.html#_npm_dependencies_and_bundling ![image](https://github.com/user-attachments/assets/09c7312b-5222-4e0e-a4b2-42cd36ada087) --- example/extending/typescript/4-npm-deps-bundle/build.mill | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/extending/typescript/4-npm-deps-bundle/build.mill b/example/extending/typescript/4-npm-deps-bundle/build.mill index d276c659e8f..1fa764480b8 100644 --- a/example/extending/typescript/4-npm-deps-bundle/build.mill +++ b/example/extending/typescript/4-npm-deps-bundle/build.mill @@ -4,7 +4,7 @@ // install `typescript` and `@types/node`, because that was what was needed // to compile Typescript against the builtin Node.js APIs. In this example, // we add a `def npmDeps` task, that is aggregated using `Task.traverse` into -// ` def transitiveNpmDeps`, that are then included in the body of `def npmInstall`. +// `def transitiveNpmDeps`, that are then included in the body of `def npmInstall`. // The `npmInstall` destination folder in then used both in `def compile` to // provide the `tsc` compiler and supporting installed type definitions, as well // as in `def run` in order to provide the necessary files to the `node` runtime.