Skip to content

Commit

Permalink
Break out AssemblyModule from JavaModule, move launcher into `R…
Browse files Browse the repository at this point in the history
…unModule` (com-lihaoyi#4301)
  • Loading branch information
lihaoyi authored and vaslabs committed Jan 13, 2025
1 parent da4fbb0 commit add26e1
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 105 deletions.
4 changes: 4 additions & 0 deletions ci/test-mill-bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ git stash pop "$(git stash list | grep "preserve mill-release" | head -n1 | sed
# Prepare local build
ci/patch-mill-bootstrap.sh

# Start clean to rule out cache invalidation issues
rm -rf out


# Run tests
./mill-assembly.jar -i "__.compile"
./mill-assembly.jar -i "example.scalalib.basic[1-simple].packaged.server.test"
4 changes: 2 additions & 2 deletions example/cli/builtins/1-builtin-commands/build.mill
Original file line number Diff line number Diff line change
Expand Up @@ -220,8 +220,8 @@ foo.sources
foo.allSources
foo.allSourceFiles
foo.compile
foo.finalMainClassOpt
foo.prependShellScript
foo.localRunClasspath
foo.localClasspath
foo.assembly

*/
Expand Down
7 changes: 4 additions & 3 deletions example/fundamentals/out-dir/1-out-files/build.mill
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,7 @@ out/mill-server

> cat out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json
{
"call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {},
"call scala.Predef$#println(java.lang.Object)void": {
"call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {
"def build_.package_$foo$#<init>(build_.package_)void": {
"call build_.package_$foo$!<init>(build_.package_)void": {
"def build_.package_#foo$lzycompute$1()void": {
Expand All @@ -254,8 +253,10 @@ out/mill-server
}
}
}
}
},
"call scala.Predef$#println(java.lang.Object)void": {}
}

*/

// In the `spanningInvalidationTree.json` above, we can see how to addition of the call
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ static List<String> millLaunchJvmCommand(boolean setJnaNoSys) throws Exception {
vmOptions.addAll(Util.readOptsFileLines(millJvmOptsFile));
}

vmOptions.add("-XX:+HeapDumpOnOutOfMemoryError");
vmOptions.add("-cp");
vmOptions.add(String.join(File.pathSeparator, millClasspath()));

Expand Down
136 changes: 136 additions & 0 deletions scalalib/src/mill/scalalib/AssemblyModule.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package mill.scalalib

import mill.api.Loose.Agg
import mill.api.{JarManifest, PathRef, Result}
import mill.define.{Target => T, _}
import mill.util.Jvm

import scala.annotation.nowarn

/**
* Core configuration required to compile a single Java compilation target
*/
trait AssemblyModule extends mill.Module {
outer =>

def finalMainClassOpt: T[Either[String, String]]

def forkArgs: T[Seq[String]]

/**
* Creates a manifest representation which can be modified or replaced
* The default implementation just adds the `Manifest-Version`, `Main-Class` and `Created-By` attributes
*/
def manifest: T[JarManifest] = Task { manifest0() }

private[mill] def manifest0: T[JarManifest] = Task {
Jvm.createManifest(finalMainClassOpt().toOption)
}

/**
* What shell script to use to launch the executable generated by `assembly`.
* Defaults to a generic "universal" launcher that should work for Windows,
* OS-X and Linux
*/
def prependShellScript: T[String] = Task {
prependShellScript0()
}
private[mill] def prependShellScript0: T[String] = Task {
finalMainClassOpt().toOption match {
case None => ""
case Some(cls) =>
mill.util.Jvm.launcherUniversalScript(
mainClass = cls,
shellClassPath = Agg("$0"),
cmdClassPath = Agg("%~dpnx0"),
jvmArgs = forkArgs()
)
}
}

def assemblyRules: Seq[Assembly.Rule] = assemblyRules0

private[mill] def assemblyRules0: Seq[Assembly.Rule] = Assembly.defaultRules

def upstreamAssemblyClasspath: T[Agg[PathRef]]

def localClasspath: T[Seq[PathRef]]

private[mill] def upstreamAssembly2_0: T[Assembly] = Task {
Assembly.create(
destJar = T.dest / "out.jar",
inputPaths = upstreamAssemblyClasspath().map(_.path),
manifest = manifest(),
assemblyRules = assemblyRules
)
}

/**
* Build the assembly for upstream dependencies separate from the current
* classpath
*
* This should allow much faster assembly creation in the common case where
* upstream dependencies do not change
*/
def upstreamAssembly2: T[Assembly] = Task {
upstreamAssembly2_0()
}

def upstreamAssembly: T[PathRef] = Task {
T.log.error(
s"upstreamAssembly target is deprecated and should no longer used." +
s" Please make sure to use upstreamAssembly2 instead."
)
upstreamAssembly2().pathRef
}

private[mill] def assembly0: Task[PathRef] = Task.Anon {
// detect potential inconsistencies due to `upstreamAssembly` deprecation after 0.11.7
if (
(upstreamAssembly.ctx.enclosing: @nowarn) != s"${classOf[AssemblyModule].getName}#upstreamAssembly"
) {
T.log.error(
s"${upstreamAssembly.ctx.enclosing: @nowarn} is overriding a deprecated target which is no longer used." +
s" Please make sure to override upstreamAssembly2 instead."
)
}

val prependScript = Option(prependShellScript()).filter(_ != "")
val upstream = upstreamAssembly2()

val created = Assembly.create(
destJar = T.dest / "out.jar",
Agg.from(localClasspath().map(_.path)),
manifest(),
prependScript,
Some(upstream.pathRef.path),
assemblyRules
)
// See https://github.com/com-lihaoyi/mill/pull/2655#issuecomment-1672468284
val problematicEntryCount = 65535
if (
prependScript.isDefined &&
(upstream.addedEntries + created.addedEntries) > problematicEntryCount
) {
Result.Failure(
s"""The created assembly jar contains more than ${problematicEntryCount} ZIP entries.
|JARs of that size are known to not work correctly with a prepended shell script.
|Either reduce the entries count of the assembly or disable the prepended shell script with:
|
| def prependShellScript = ""
|""".stripMargin,
Some(created.pathRef)
)
} else {
Result.Success(created.pathRef)
}
}

/**
* An executable uber-jar/assembly containing all the resources and compiled
* classfiles from this module and all it's upstream modules and dependencies
*/
def assembly: T[PathRef] = T {
assembly0()
}
}
111 changes: 11 additions & 100 deletions scalalib/src/mill/scalalib/JavaModule.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import mill.util.Jvm
import os.{Path, ProcessOutput}

import scala.annotation.nowarn
import mill.define.Target

/**
* Core configuration required to compile a single Java compilation target
Expand All @@ -34,7 +33,8 @@ trait JavaModule
with CoursierModule
with OfflineSupportModule
with BspModule
with SemanticDbJavaModule { outer =>
with SemanticDbJavaModule
with AssemblyModule { outer =>

override def zincWorker: ModuleRef[ZincWorkerModule] = super.zincWorker
@nowarn
Expand Down Expand Up @@ -575,22 +575,9 @@ trait JavaModule
*/
def platformSuffix: T[String] = Task { "" }

/**
* What shell script to use to launch the executable generated by `assembly`.
* Defaults to a generic "universal" launcher that should work for Windows,
* OS-X and Linux
*/
// bincompat stub
def prependShellScript: T[String] = Task {
finalMainClassOpt().toOption match {
case None => ""
case Some(cls) =>
mill.util.Jvm.launcherUniversalScript(
mainClass = cls,
shellClassPath = Agg("$0"),
cmdClassPath = Agg("%~dpnx0"),
jvmArgs = forkArgs()
)
}
prependShellScript0()
}

/**
Expand Down Expand Up @@ -811,13 +798,8 @@ trait JavaModule
localClasspath()
}

/**
* Creates a manifest representation which can be modified or replaced
* The default implementation just adds the `Manifest-Version`, `Main-Class` and `Created-By` attributes
*/
def manifest: T[JarManifest] = Task {
Jvm.createManifest(finalMainClassOpt().toOption)
}
// bincompat stub
def manifest: T[JarManifest] = Task { manifest0() }

/**
* Build the assembly for upstream dependencies separate from the current
Expand All @@ -838,67 +820,11 @@ trait JavaModule
upstreamAssembly2().pathRef
}

/**
* Build the assembly for upstream dependencies separate from the current
* classpath
*
* This should allow much faster assembly creation in the common case where
* upstream dependencies do not change
*/
def upstreamAssembly2: T[Assembly] = Task {
Assembly.create(
destJar = T.dest / "out.jar",
inputPaths = upstreamAssemblyClasspath().map(_.path),
manifest = manifest(),
assemblyRules = assemblyRules
)
}

/**
* An executable uber-jar/assembly containing all the resources and compiled
* classfiles from this module and all it's upstream modules and dependencies
*/
def assembly: T[PathRef] = Task {
// detect potential inconsistencies due to `upstreamAssembly` deprecation after 0.11.7
if (
(upstreamAssembly.ctx.enclosing: @nowarn) != s"${classOf[JavaModule].getName}#upstreamAssembly"
) {
T.log.error(
s"${upstreamAssembly.ctx.enclosing: @nowarn} is overriding a deprecated target which is no longer used." +
s" Please make sure to override upstreamAssembly2 instead."
)
}

val prependScript = Option(prependShellScript()).filter(_ != "")
val upstream = upstreamAssembly2()
// Bincompat stub
def upstreamAssembly2: T[Assembly] = Task { upstreamAssembly2_0() }

val created = Assembly.create(
destJar = T.dest / "out.jar",
Agg.from(localClasspath().map(_.path)),
manifest(),
prependScript,
Some(upstream.pathRef.path),
assemblyRules
)
// See https://github.com/com-lihaoyi/mill/pull/2655#issuecomment-1672468284
val problematicEntryCount = 65535
if (
prependScript.isDefined &&
(upstream.addedEntries + created.addedEntries) > problematicEntryCount
) {
Result.Failure(
s"""The created assembly jar contains more than ${problematicEntryCount} ZIP entries.
|JARs of that size are known to not work correctly with a prepended shell script.
|Either reduce the entries count of the assembly or disable the prepended shell script with:
|
| def prependShellScript = ""
|""".stripMargin,
Some(created.pathRef)
)
} else {
Result.Success(created.pathRef)
}
}
// Bincompat stub
override def assembly: T[PathRef] = Task[PathRef] { assembly0() }

/**
* A jar containing only this module's resources and compiled classfiles,
Expand Down Expand Up @@ -1025,22 +951,7 @@ trait JavaModule
super.forkEnv()
}

/**
* Builds a command-line "launcher" file that can be used to run this module's
* code, without the Mill process. Useful for deployment & other places where
* you do not want a build tool running
*/
def launcher: Target[PathRef] = Task {
val launchClasspath =
if (!runUseArgsFile()) runClasspath().map(_.path)
else {
val classpathJar = Task.dest / "classpath.jar"
Jvm.createClasspathPassingJar(classpathJar, runClasspath().map(_.path))
Agg(classpathJar)
}

Jvm.createLauncher(finalMainClass(), launchClasspath, forkArgs())
}
def launcher: T[PathRef] = Task { launcher0() }

/**
* Task that print the transitive dependency tree to STDOUT.
Expand Down
19 changes: 19 additions & 0 deletions scalalib/src/mill/scalalib/RunModule.scala
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,25 @@ trait RunModule extends WithZincWorker {
Result.Failure("subprocess failed")
}
}

private[mill] def launcher0 = Task.Anon {
val launchClasspath =
if (!runUseArgsFile()) runClasspath().map(_.path)
else {
val classpathJar = Task.dest / "classpath.jar"
Jvm.createClasspathPassingJar(classpathJar, runClasspath().map(_.path))
Agg(classpathJar)
}

Jvm.createLauncher(finalMainClass(), launchClasspath, forkArgs())
}

/**
* Builds a command-line "launcher" file that can be used to run this module's
* code, without the Mill process. Useful for deployment & other places where
* you do not want a build tool running
*/
def launcher = Task { launcher0() }
}

object RunModule {
Expand Down

0 comments on commit add26e1

Please sign in to comment.