Skip to content

Commit

Permalink
.
Browse files Browse the repository at this point in the history
  • Loading branch information
lihaoyi committed Feb 11, 2025
1 parent 6314156 commit 07c74dd
Show file tree
Hide file tree
Showing 10 changed files with 107 additions and 124 deletions.
41 changes: 24 additions & 17 deletions core/exec/src/mill/exec/Execution.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,25 @@ import scala.concurrent._
/**
* Core logic of evaluating tasks, without any user-facing helper methods
*/
private[mill] class Execution(val baseLogger: ColorLogger,
val chromeProfileLogger: ChromeProfileLogger,
val profileLogger: ProfileLogger,
val home: os.Path,
val workspace: os.Path,
val outPath: os.Path,
val externalOutPath: os.Path,
val rootModule: BaseModule,
val classLoaderSigHash: Int,
val classLoaderIdentityHash: Int,
val workerCache: mutable.Map[Segments, (Int, Val)],
val env: Map[String, String],
val failFast: Boolean,
val threadCount: Option[Int],
val methodCodeHashSignatures: Map[String, Int],
val systemExit: Int => Nothing,
val exclusiveSystemStreams: SystemStreams) extends GroupExecution {
private[mill] case class Execution(val baseLogger: ColorLogger,
val chromeProfileLogger: ChromeProfileLogger,
val profileLogger: ProfileLogger,
val home: os.Path,
val workspace: os.Path,
val outPath: os.Path,
val externalOutPath: os.Path,
val rootModule: BaseModule,
val classLoaderSigHash: Int,
val classLoaderIdentityHash: Int,
val workerCache: mutable.Map[Segments, (Int, Val)],
val env: Map[String, String],
val failFast: Boolean,
val threadCount: Option[Int],
val methodCodeHashSignatures: Map[String, Int],
val systemExit: Int => Nothing,
val exclusiveSystemStreams: SystemStreams) extends GroupExecution with AutoCloseable{

def withBaseLogger(newBaseLogger: ColorLogger) = this.copy(baseLogger = newBaseLogger)

/**
* @param goals The tasks that need to be evaluated
Expand Down Expand Up @@ -266,6 +268,11 @@ private[mill] class Execution(val baseLogger: ColorLogger,
results.map { case (k, v) => (k, v.map(_._1)) }
)
}

def close(): Unit = {
chromeProfileLogger.close()
profileLogger.close()
}
}

private[mill] object Execution {
Expand Down
6 changes: 3 additions & 3 deletions core/exec/test/src/mill/exec/Checker.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import utest.*
class Checker[T <: mill.testkit.TestBaseModule](module: T, threadCount: Option[Int] = Some(1)) {
// Make sure data is persisted even if we re-create the evaluator each time

val evaluator = UnitTester(module, null, threads = threadCount).evaluator
val execution = UnitTester(module, null, threads = threadCount).execution

def apply(
target: Task[?],
Expand All @@ -23,7 +23,7 @@ class Checker[T <: mill.testkit.TestBaseModule](module: T, threadCount: Option[I
secondRunNoOp: Boolean = true
) = {

val evaled = evaluator.execution.executeTasks(Seq(target))
val evaled = execution.executeTasks(Seq(target))

val (matchingReturnedEvaled, extra) =
evaled.evaluated.partition(expEvaled.contains)
Expand All @@ -36,7 +36,7 @@ class Checker[T <: mill.testkit.TestBaseModule](module: T, threadCount: Option[I

// Second time the value is already cached, so no evaluation needed
if (secondRunNoOp) {
val evaled2 = evaluator.execution.executeTasks(Seq(target))
val evaled2 = execution.executeTasks(Seq(target))
val expectedSecondRunEvaluated = Seq()
assert(
evaled2.values.map(_.value) == evaled.values.map(_.value),
Expand Down
6 changes: 3 additions & 3 deletions core/exec/test/src/mill/exec/EvaluationTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -162,14 +162,14 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite {

import separateGroups._
val checker = new Checker(separateGroups)
val evaled1 = checker.evaluator.execution.executeTasks(Seq(right, left))
val evaled1 = checker.execution.executeTasks(Seq(right, left))
val filtered1 = evaled1.evaluated.filter(_.isInstanceOf[TargetImpl[?]])
assert(filtered1.toSeq.sortBy(_.toString) == Seq(change, left, right).sortBy(_.toString))
val evaled2 = checker.evaluator.execution.executeTasks(Seq(right, left))
val evaled2 = checker.execution.executeTasks(Seq(right, left))
val filtered2 = evaled2.evaluated.filter(_.isInstanceOf[TargetImpl[?]])
assert(filtered2 == Seq())
change.counter += 1
val evaled3 = checker.evaluator.execution.executeTasks(Seq(right, left))
val evaled3 = checker.execution.executeTasks(Seq(right, left))
val filtered3 = evaled3.evaluated.filter(_.isInstanceOf[TargetImpl[?]])
assert(filtered3 == Seq(change, right))

Expand Down
2 changes: 1 addition & 1 deletion core/exec/test/src/mill/exec/ModuleTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ object ModuleTests extends TestSuite {
val zresult = check.apply(Build.z)
assert(
zresult == Right(Result(30, 1)),
os.read(check.evaluator.outPath / "z.json").contains("30"),
os.read(check.execution.outPath / "z.json").contains("30"),
os.read(
check.outPath / "mill/exec/TestExternalModule/x.json"
).contains("13"),
Expand Down
30 changes: 15 additions & 15 deletions core/exec/test/src/mill/exec/OverrideTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,9 @@ object OverrideTests extends TestSuite {
val checker = new Checker(canOverrideSuper)
checker(foo, Seq("base", "object"), Seq(foo), extraEvaled = -1)

val public = os.read(checker.evaluator.outPath / "foo.json")
val public = os.read(checker.execution.outPath / "foo.json")
val overridden = os.read(
checker.evaluator.outPath / "foo.super/BaseModule.json"
checker.execution.outPath / "foo.super/BaseModule.json"
)
assert(
public.contains("base"),
Expand All @@ -129,9 +129,9 @@ object OverrideTests extends TestSuite {
secondRunNoOp = false
)

val public = os.read(checker.evaluator.outPath / "cmd.json")
val public = os.read(checker.execution.outPath / "cmd.json")
val overridden = os.read(
checker.evaluator.outPath / "cmd.super/BaseModule.json"
checker.execution.outPath / "cmd.super/BaseModule.json"
)
assert(
public.contains("base1"),
Expand All @@ -155,14 +155,14 @@ object OverrideTests extends TestSuite {
)

assert(
os.read(checker.evaluator.outPath / "m/f.super/X.json")
os.read(checker.execution.outPath / "m/f.super/X.json")
.contains(" 1,")
)
assert(
os.read(checker.evaluator.outPath / "m/f.super/A.json")
os.read(checker.execution.outPath / "m/f.super/A.json")
.contains(" 3,")
)
assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,"))
assert(os.read(checker.execution.outPath / "m/f.json").contains(" 6,"))
}
test("stackableOverrides2") {
// When the supers have the same name, qualify them until they are distinct
Expand All @@ -177,14 +177,14 @@ object OverrideTests extends TestSuite {
)

assert(
os.read(checker.evaluator.outPath / "m/f.super/A/X.json")
os.read(checker.execution.outPath / "m/f.super/A/X.json")
.contains(" 1,")
)
assert(
os.read(checker.evaluator.outPath / "m/f.super/B/X.json")
os.read(checker.execution.outPath / "m/f.super/B/X.json")
.contains(" 3,")
)
assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,"))
assert(os.read(checker.execution.outPath / "m/f.json").contains(" 6,"))
}
test("stackableOverrides3") {
// When the supers have the same name, qualify them until they are distinct
Expand All @@ -199,14 +199,14 @@ object OverrideTests extends TestSuite {
)

assert(
os.read(checker.evaluator.outPath / "m/f.super/A/X.json")
os.read(checker.execution.outPath / "m/f.super/A/X.json")
.contains(" 1,")
)
assert(
os.read(checker.evaluator.outPath / "m/f.super/X.json")
os.read(checker.execution.outPath / "m/f.super/X.json")
.contains(" 3,")
)
assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,"))
assert(os.read(checker.execution.outPath / "m/f.json").contains(" 6,"))
}
test("optionalOverride") {
// Make sure that when a task is overriden, it always gets put in the same place on
Expand All @@ -217,14 +217,14 @@ object OverrideTests extends TestSuite {
test {
checker(m.f, 11, Seq(m.f), extraEvaled = -1)
assert(
os.read(checker.evaluator.outPath / "m/f.super/X.json")
os.read(checker.execution.outPath / "m/f.super/X.json")
.contains(" 1,")
)
}
test {
checker(m.g, 101, Seq(), extraEvaled = -1)
assert(
os.read(checker.evaluator.outPath / "m/f.super/X.json")
os.read(checker.execution.outPath / "m/f.super/X.json")
.contains(" 1,")
)
}
Expand Down
10 changes: 5 additions & 5 deletions core/exec/test/src/mill/exec/TaskTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ trait TaskTests extends TestSuite {

test("worker") {
test("static") - withEnv { (build, check) =>
val wc = check.evaluator.workerCache
val wc = check.execution.workerCache

check(build.staticWorkerDownstream) ==> Right(Result(2, 1))
wc.size ==> 1
Expand All @@ -181,10 +181,10 @@ trait TaskTests extends TestSuite {
wc.head ==> firstCached
}
test("staticButReevaluated") - withEnv { (build, check) =>
val wc = check.evaluator.workerCache
val wc = check.execution.workerCache

check(build.staticWorkerDownstreamReeval) ==> Right(Result(2, 1))
check.evaluator.workerCache.size ==> 1
check.execution.workerCache.size ==> 1
val firstCached = wc.head

check(build.staticWorkerDownstreamReeval) ==> Right(Result(2, 1))
Expand All @@ -199,7 +199,7 @@ trait TaskTests extends TestSuite {
check(build.changeOnceWorkerDownstream) ==> Right(Result(2, 0))
}
test("alwaysChanged") - withEnv { (build, check) =>
val wc = check.evaluator.workerCache
val wc = check.execution.workerCache

check(build.noisyWorkerDownstream) ==> Right(Result(2, 1))
wc.size ==> 1
Expand All @@ -215,7 +215,7 @@ trait TaskTests extends TestSuite {
assert(wc.head != secondCached)
}
test("closableWorker") - withEnv { (build, check) =>
val wc = check.evaluator.workerCache
val wc = check.execution.workerCache

check(build.noisyClosableWorkerDownstream) ==> Right(Result(2, 1))
wc.size ==> 1
Expand Down
64 changes: 16 additions & 48 deletions core/src/mill/eval/Evaluator.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import OutFiles.*
import mill.resolve.Resolve

import scala.jdk.CollectionConverters.*
import scala.collection.mutable
import scala.util.DynamicVariable

/**
Expand All @@ -28,55 +27,25 @@ import scala.util.DynamicVariable
* extracted into [[Execution]]
*/
final case class Evaluator private[mill] (
home: os.Path,
workspace: os.Path,
outPath: os.Path,
externalOutPath: os.Path,
rootModule: mill.define.BaseModule,
baseLogger: ColorLogger,
classLoaderSigHash: Int,
classLoaderIdentityHash: Int,
workerCache: mutable.Map[Segments, (Int, Val)] = mutable.Map.empty,
env: Map[String, String],
failFast: Boolean,
threadCount: Option[Int],
methodCodeHashSignatures: Map[String, Int],
allowPositionalCommandArgs: Boolean,
systemExit: Int => Nothing,
exclusiveSystemStreams: SystemStreams,
selectiveExecution: Boolean = false,
chromeProfileLogger: ChromeProfileLogger,
profileLogger: ProfileLogger
private[mill] val allowPositionalCommandArgs: Boolean,
private[mill] val selectiveExecution: Boolean = false,
private[mill] val execution: Execution
) extends AutoCloseable {
private[mill] val execution = new Execution(
baseLogger,
chromeProfileLogger,
profileLogger,
home,
workspace,
outPath,
externalOutPath,
rootModule,
classLoaderSigHash,
classLoaderIdentityHash,
workerCache,
env,
failFast,
threadCount,
methodCodeHashSignatures,
systemExit,
exclusiveSystemStreams,
)
import Evaluator._

private[mill] final def mutableWorkerCache: collection.mutable.Map[Segments, (Int, Val)] =
workerCache match {
case mut: collection.mutable.Map[Segments, (Int, Val)] => mut
}
private[mill] def workspace = execution.workspace
private[mill] def baseLogger = execution.baseLogger
private[mill] def outPath = execution.outPath
private[mill] def methodCodeHashSignatures = execution.methodCodeHashSignatures
private[mill] def rootModule = execution.rootModule
private[mill] def workerCache = execution.workerCache

val pathsResolver: ExecutionPathsResolver = ExecutionPathsResolver.default(outPath)

def withBaseLogger(newBaseLogger: ColorLogger): Evaluator =
this.copy(baseLogger = newBaseLogger)
def withBaseLogger(newBaseLogger: ColorLogger): Evaluator = new Evaluator(
allowPositionalCommandArgs,
selectiveExecution,
execution.withBaseLogger(newBaseLogger)
)

def plan(goals: Seq[Task[?]]): Plan = Plan.plan(goals)

Expand Down Expand Up @@ -111,8 +80,7 @@ final case class Evaluator private[mill] (
}

def close(): Unit = {
chromeProfileLogger.close()
profileLogger.close()
execution.close()
}

def resolveEvaluate(
Expand Down
2 changes: 1 addition & 1 deletion main/src/mill/main/MainModule.scala
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ trait MainModule extends BaseModule {
workerSegments <- evaluator.workerCache.keys.toList
if allSegments.exists(workerSegments.startsWith)
case (_, Val(closeable: AutoCloseable)) <-
evaluator.mutableWorkerCache.remove(workerSegments)
evaluator.execution.workerCache.remove(workerSegments)
} {
closeable.close()
}
Expand Down
39 changes: 21 additions & 18 deletions runner/src/mill/runner/MillBuildBootstrap.scala
Original file line number Diff line number Diff line change
Expand Up @@ -349,26 +349,29 @@ class MillBuildBootstrap(
)

val outPath = recOut(output, depth)
val baseLogger = new PrefixLogger(logger, bootLogPrefix)
new mill.eval.Evaluator(
home,
projectRoot,
outPath,
outPath,
rootModule,
new PrefixLogger(logger, bootLogPrefix),
classLoaderSigHash = millClassloaderSigHash,
classLoaderIdentityHash = millClassloaderIdentityHash,
workerCache = workerCache.to(collection.mutable.Map),
env = env,
failFast = !keepGoing,
threadCount = threadCount,
methodCodeHashSignatures = methodCodeHashSignatures,
allowPositionalCommandArgs = allowPositionalCommandArgs,
systemExit = systemExit,
exclusiveSystemStreams = streams0,
selectiveExecution = selectiveExecution,
chromeProfileLogger = new ChromeProfileLogger(outPath / millChromeProfile),
profileLogger = new ProfileLogger(outPath / millProfile)
selectiveExecution = false,
execution = new mill.exec.Execution(
baseLogger = baseLogger,
chromeProfileLogger = new ChromeProfileLogger(outPath / millChromeProfile),
profileLogger = new ProfileLogger(outPath / millProfile),
home = home,
workspace = projectRoot,
outPath = outPath,
externalOutPath = outPath,
rootModule = rootModule,
classLoaderSigHash = millClassloaderSigHash,
classLoaderIdentityHash = millClassloaderIdentityHash,
workerCache = workerCache.to(collection.mutable.Map),
env = env,
failFast = !keepGoing,
threadCount = threadCount,
methodCodeHashSignatures = methodCodeHashSignatures,
systemExit = systemExit,
exclusiveSystemStreams = streams0,
)
)
}

Expand Down
Loading

0 comments on commit 07c74dd

Please sign in to comment.