From add26e1aeaf89cf6ca613437177b2e5191d942d8 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 13 Jan 2025 22:05:03 +0800 Subject: [PATCH] Break out `AssemblyModule` from `JavaModule`, move `launcher` into `RunModule` (#4301) --- ci/test-mill-bootstrap.sh | 4 + .../builtins/1-builtin-commands/build.mill | 4 +- .../out-dir/1-out-files/build.mill | 7 +- .../runner/client/MillProcessLauncher.java | 1 + .../src/mill/scalalib/AssemblyModule.scala | 136 ++++++++++++++++++ scalalib/src/mill/scalalib/JavaModule.scala | 111 ++------------ scalalib/src/mill/scalalib/RunModule.scala | 19 +++ 7 files changed, 177 insertions(+), 105 deletions(-) create mode 100644 scalalib/src/mill/scalalib/AssemblyModule.scala diff --git a/ci/test-mill-bootstrap.sh b/ci/test-mill-bootstrap.sh index efaee1aeacff..47beeb993382 100755 --- a/ci/test-mill-bootstrap.sh +++ b/ci/test-mill-bootstrap.sh @@ -18,6 +18,10 @@ git stash pop "$(git stash list | grep "preserve mill-release" | head -n1 | sed # Prepare local build ci/patch-mill-bootstrap.sh +# Start clean to rule out cache invalidation issues +rm -rf out + + # Run tests ./mill-assembly.jar -i "__.compile" ./mill-assembly.jar -i "example.scalalib.basic[1-simple].packaged.server.test" diff --git a/example/cli/builtins/1-builtin-commands/build.mill b/example/cli/builtins/1-builtin-commands/build.mill index 4caa954edeb6..262a16ca10ee 100644 --- a/example/cli/builtins/1-builtin-commands/build.mill +++ b/example/cli/builtins/1-builtin-commands/build.mill @@ -220,8 +220,8 @@ foo.sources foo.allSources foo.allSourceFiles foo.compile -foo.finalMainClassOpt -foo.prependShellScript +foo.localRunClasspath +foo.localClasspath foo.assembly */ diff --git a/example/fundamentals/out-dir/1-out-files/build.mill b/example/fundamentals/out-dir/1-out-files/build.mill index e67d719df6ad..3aa97215e778 100644 --- a/example/fundamentals/out-dir/1-out-files/build.mill +++ b/example/fundamentals/out-dir/1-out-files/build.mill @@ -243,8 +243,7 @@ out/mill-server > cat out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json { - "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {}, - "call scala.Predef$#println(java.lang.Object)void": { + "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": { "def build_.package_$foo$#(build_.package_)void": { "call build_.package_$foo$!(build_.package_)void": { "def build_.package_#foo$lzycompute$1()void": { @@ -254,8 +253,10 @@ out/mill-server } } } - } + }, + "call scala.Predef$#println(java.lang.Object)void": {} } + */ // In the `spanningInvalidationTree.json` above, we can see how to addition of the call diff --git a/runner/client/src/mill/runner/client/MillProcessLauncher.java b/runner/client/src/mill/runner/client/MillProcessLauncher.java index b2b33eeba5c4..9eff9a105b20 100644 --- a/runner/client/src/mill/runner/client/MillProcessLauncher.java +++ b/runner/client/src/mill/runner/client/MillProcessLauncher.java @@ -217,6 +217,7 @@ static List millLaunchJvmCommand(boolean setJnaNoSys) throws Exception { vmOptions.addAll(Util.readOptsFileLines(millJvmOptsFile)); } + vmOptions.add("-XX:+HeapDumpOnOutOfMemoryError"); vmOptions.add("-cp"); vmOptions.add(String.join(File.pathSeparator, millClasspath())); diff --git a/scalalib/src/mill/scalalib/AssemblyModule.scala b/scalalib/src/mill/scalalib/AssemblyModule.scala new file mode 100644 index 000000000000..9cb1a9bc7ddb --- /dev/null +++ b/scalalib/src/mill/scalalib/AssemblyModule.scala @@ -0,0 +1,136 @@ +package mill.scalalib + +import mill.api.Loose.Agg +import mill.api.{JarManifest, PathRef, Result} +import mill.define.{Target => T, _} +import mill.util.Jvm + +import scala.annotation.nowarn + +/** + * Core configuration required to compile a single Java compilation target + */ +trait AssemblyModule extends mill.Module { + outer => + + def finalMainClassOpt: T[Either[String, String]] + + def forkArgs: T[Seq[String]] + + /** + * Creates a manifest representation which can be modified or replaced + * The default implementation just adds the `Manifest-Version`, `Main-Class` and `Created-By` attributes + */ + def manifest: T[JarManifest] = Task { manifest0() } + + private[mill] def manifest0: T[JarManifest] = Task { + Jvm.createManifest(finalMainClassOpt().toOption) + } + + /** + * What shell script to use to launch the executable generated by `assembly`. + * Defaults to a generic "universal" launcher that should work for Windows, + * OS-X and Linux + */ + def prependShellScript: T[String] = Task { + prependShellScript0() + } + private[mill] def prependShellScript0: T[String] = Task { + finalMainClassOpt().toOption match { + case None => "" + case Some(cls) => + mill.util.Jvm.launcherUniversalScript( + mainClass = cls, + shellClassPath = Agg("$0"), + cmdClassPath = Agg("%~dpnx0"), + jvmArgs = forkArgs() + ) + } + } + + def assemblyRules: Seq[Assembly.Rule] = assemblyRules0 + + private[mill] def assemblyRules0: Seq[Assembly.Rule] = Assembly.defaultRules + + def upstreamAssemblyClasspath: T[Agg[PathRef]] + + def localClasspath: T[Seq[PathRef]] + + private[mill] def upstreamAssembly2_0: T[Assembly] = Task { + Assembly.create( + destJar = T.dest / "out.jar", + inputPaths = upstreamAssemblyClasspath().map(_.path), + manifest = manifest(), + assemblyRules = assemblyRules + ) + } + + /** + * Build the assembly for upstream dependencies separate from the current + * classpath + * + * This should allow much faster assembly creation in the common case where + * upstream dependencies do not change + */ + def upstreamAssembly2: T[Assembly] = Task { + upstreamAssembly2_0() + } + + def upstreamAssembly: T[PathRef] = Task { + T.log.error( + s"upstreamAssembly target is deprecated and should no longer used." + + s" Please make sure to use upstreamAssembly2 instead." + ) + upstreamAssembly2().pathRef + } + + private[mill] def assembly0: Task[PathRef] = Task.Anon { + // detect potential inconsistencies due to `upstreamAssembly` deprecation after 0.11.7 + if ( + (upstreamAssembly.ctx.enclosing: @nowarn) != s"${classOf[AssemblyModule].getName}#upstreamAssembly" + ) { + T.log.error( + s"${upstreamAssembly.ctx.enclosing: @nowarn} is overriding a deprecated target which is no longer used." + + s" Please make sure to override upstreamAssembly2 instead." + ) + } + + val prependScript = Option(prependShellScript()).filter(_ != "") + val upstream = upstreamAssembly2() + + val created = Assembly.create( + destJar = T.dest / "out.jar", + Agg.from(localClasspath().map(_.path)), + manifest(), + prependScript, + Some(upstream.pathRef.path), + assemblyRules + ) + // See https://github.com/com-lihaoyi/mill/pull/2655#issuecomment-1672468284 + val problematicEntryCount = 65535 + if ( + prependScript.isDefined && + (upstream.addedEntries + created.addedEntries) > problematicEntryCount + ) { + Result.Failure( + s"""The created assembly jar contains more than ${problematicEntryCount} ZIP entries. + |JARs of that size are known to not work correctly with a prepended shell script. + |Either reduce the entries count of the assembly or disable the prepended shell script with: + | + | def prependShellScript = "" + |""".stripMargin, + Some(created.pathRef) + ) + } else { + Result.Success(created.pathRef) + } + } + + /** + * An executable uber-jar/assembly containing all the resources and compiled + * classfiles from this module and all it's upstream modules and dependencies + */ + def assembly: T[PathRef] = T { + assembly0() + } +} diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 1c341afa7b38..4e12b89fd3b7 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -19,7 +19,6 @@ import mill.util.Jvm import os.{Path, ProcessOutput} import scala.annotation.nowarn -import mill.define.Target /** * Core configuration required to compile a single Java compilation target @@ -34,7 +33,8 @@ trait JavaModule with CoursierModule with OfflineSupportModule with BspModule - with SemanticDbJavaModule { outer => + with SemanticDbJavaModule + with AssemblyModule { outer => override def zincWorker: ModuleRef[ZincWorkerModule] = super.zincWorker @nowarn @@ -575,22 +575,9 @@ trait JavaModule */ def platformSuffix: T[String] = Task { "" } - /** - * What shell script to use to launch the executable generated by `assembly`. - * Defaults to a generic "universal" launcher that should work for Windows, - * OS-X and Linux - */ + // bincompat stub def prependShellScript: T[String] = Task { - finalMainClassOpt().toOption match { - case None => "" - case Some(cls) => - mill.util.Jvm.launcherUniversalScript( - mainClass = cls, - shellClassPath = Agg("$0"), - cmdClassPath = Agg("%~dpnx0"), - jvmArgs = forkArgs() - ) - } + prependShellScript0() } /** @@ -811,13 +798,8 @@ trait JavaModule localClasspath() } - /** - * Creates a manifest representation which can be modified or replaced - * The default implementation just adds the `Manifest-Version`, `Main-Class` and `Created-By` attributes - */ - def manifest: T[JarManifest] = Task { - Jvm.createManifest(finalMainClassOpt().toOption) - } + // bincompat stub + def manifest: T[JarManifest] = Task { manifest0() } /** * Build the assembly for upstream dependencies separate from the current @@ -838,67 +820,11 @@ trait JavaModule upstreamAssembly2().pathRef } - /** - * Build the assembly for upstream dependencies separate from the current - * classpath - * - * This should allow much faster assembly creation in the common case where - * upstream dependencies do not change - */ - def upstreamAssembly2: T[Assembly] = Task { - Assembly.create( - destJar = T.dest / "out.jar", - inputPaths = upstreamAssemblyClasspath().map(_.path), - manifest = manifest(), - assemblyRules = assemblyRules - ) - } - - /** - * An executable uber-jar/assembly containing all the resources and compiled - * classfiles from this module and all it's upstream modules and dependencies - */ - def assembly: T[PathRef] = Task { - // detect potential inconsistencies due to `upstreamAssembly` deprecation after 0.11.7 - if ( - (upstreamAssembly.ctx.enclosing: @nowarn) != s"${classOf[JavaModule].getName}#upstreamAssembly" - ) { - T.log.error( - s"${upstreamAssembly.ctx.enclosing: @nowarn} is overriding a deprecated target which is no longer used." + - s" Please make sure to override upstreamAssembly2 instead." - ) - } - - val prependScript = Option(prependShellScript()).filter(_ != "") - val upstream = upstreamAssembly2() + // Bincompat stub + def upstreamAssembly2: T[Assembly] = Task { upstreamAssembly2_0() } - val created = Assembly.create( - destJar = T.dest / "out.jar", - Agg.from(localClasspath().map(_.path)), - manifest(), - prependScript, - Some(upstream.pathRef.path), - assemblyRules - ) - // See https://github.com/com-lihaoyi/mill/pull/2655#issuecomment-1672468284 - val problematicEntryCount = 65535 - if ( - prependScript.isDefined && - (upstream.addedEntries + created.addedEntries) > problematicEntryCount - ) { - Result.Failure( - s"""The created assembly jar contains more than ${problematicEntryCount} ZIP entries. - |JARs of that size are known to not work correctly with a prepended shell script. - |Either reduce the entries count of the assembly or disable the prepended shell script with: - | - | def prependShellScript = "" - |""".stripMargin, - Some(created.pathRef) - ) - } else { - Result.Success(created.pathRef) - } - } + // Bincompat stub + override def assembly: T[PathRef] = Task[PathRef] { assembly0() } /** * A jar containing only this module's resources and compiled classfiles, @@ -1025,22 +951,7 @@ trait JavaModule super.forkEnv() } - /** - * Builds a command-line "launcher" file that can be used to run this module's - * code, without the Mill process. Useful for deployment & other places where - * you do not want a build tool running - */ - def launcher: Target[PathRef] = Task { - val launchClasspath = - if (!runUseArgsFile()) runClasspath().map(_.path) - else { - val classpathJar = Task.dest / "classpath.jar" - Jvm.createClasspathPassingJar(classpathJar, runClasspath().map(_.path)) - Agg(classpathJar) - } - - Jvm.createLauncher(finalMainClass(), launchClasspath, forkArgs()) - } + def launcher: T[PathRef] = Task { launcher0() } /** * Task that print the transitive dependency tree to STDOUT. diff --git a/scalalib/src/mill/scalalib/RunModule.scala b/scalalib/src/mill/scalalib/RunModule.scala index 04cd5f41505b..28cfcdb5328d 100644 --- a/scalalib/src/mill/scalalib/RunModule.scala +++ b/scalalib/src/mill/scalalib/RunModule.scala @@ -231,6 +231,25 @@ trait RunModule extends WithZincWorker { Result.Failure("subprocess failed") } } + + private[mill] def launcher0 = Task.Anon { + val launchClasspath = + if (!runUseArgsFile()) runClasspath().map(_.path) + else { + val classpathJar = Task.dest / "classpath.jar" + Jvm.createClasspathPassingJar(classpathJar, runClasspath().map(_.path)) + Agg(classpathJar) + } + + Jvm.createLauncher(finalMainClass(), launchClasspath, forkArgs()) + } + + /** + * Builds a command-line "launcher" file that can be used to run this module's + * code, without the Mill process. Useful for deployment & other places where + * you do not want a build tool running + */ + def launcher = Task { launcher0() } } object RunModule {