From 2397018732ea58cb6e90e5c8ee01f4806db50392 Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 10:56:23 -0700 Subject: [PATCH 1/6] Some initial 'each' refactoring. --- src/lein_monolith/task/each.clj | 129 ++++++++++++++++---------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index 562478c..3f0c4eb 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -25,6 +25,8 @@ java.io.OutputStream)) +;; ## Task Options + (def task-opts (merge target/selection-opts @@ -74,55 +76,8 @@ [:start start]))) -(defn- print-report - "Reports information about the tasks given a results map." - [results elapsed] - (let [task-time (reduce + (keep :elapsed results)) - speedup (/ task-time elapsed)] - (lein/info (format "\n%s %11s" - (colorize [:bold :cyan] "Run time:") - (u/human-duration elapsed))) - (lein/info (format "%s %11s" - (colorize [:bold :cyan] "Task time:") - (u/human-duration task-time))) - (lein/info (format "%s %11.1f" - (colorize [:bold :cyan] "Speedup:") - speedup)) - (lein/info (->> results - (sort-by :elapsed) - (reverse) - (take 8) - (map #(format "%-45s %s %11s" - (colorize [:bold :yellow] (:name %)) - (if (:success %) " " "!") - (u/human-duration (:elapsed %)))) - (str/join "\n") - (str \newline - (colorize [:bold :cyan] "Slowest projects:") - \newline))))) - - -(defn- select-projects - "Returns a vector of pairs of index numbers and symbols naming the selected - subprojects." - [monolith subprojects fprints opts] - (let [dependencies (dep/dependency-map subprojects) - targets (target/select monolith subprojects opts) - start-from (some->> (:start opts) - (read-string) - (dep/resolve-name! (keys subprojects))) - marker (:changed opts)] - (-> - ; Sort project names by dependency order. - (dep/topological-sort dependencies targets) - (cond->> - ; Skip projects until the starting project, if provided. - start-from (drop-while (partial not= start-from)) - ; Skip projects whose fingerprint hasn't changed. - marker (filter (partial fingerprint/changed? fprints marker))) - ; Pair names up with an index [[i project-sym] ...] - (->> (map-indexed vector))))) +;; ## Output Handling (defn- tee-output-stream "Constructs a proxy of an OutputStream that will write a copy of the bytes @@ -273,7 +228,7 @@ (defn- run-task! "Runs the given task, returning a map of information about the run." [ctx target] - ; Try to reclaim some memory before running the task. + ;; Try to reclaim some memory before running the task. (System/gc) (let [start (System/nanoTime) opts (:opts ctx) @@ -289,9 +244,9 @@ (str " (" (fingerprint/explain-str fprints marker target) ")") ""))) (if-let [out-dir (get-in ctx [:opts :output])] - ; Capture output to file. + ;; Capture output to file. (apply-subproject-task-with-output subproject (:task ctx) out-dir results) - ; Run without output capturing. + ;; Run without output capturing. (apply-subproject-task subproject (:task ctx))) (when (:refresh opts) (fingerprint/save! fprints marker target) @@ -334,8 +289,9 @@ (mapv (comp (partial run-task! ctx) second) targets)) -(defn- run-parallel* - "Internal helper for `run-parallel!` which sets up the actual project threads." +(defn- run-parallel! + "Runs the tasks for targets in multiple worker threads, chained by dependency + order. Returns a vector of result maps in the order the tasks finished executing." [ctx threads targets] (let [deps (partial dep/upstream-keys (dep/dependency-map (:subprojects ctx))) thread-pool (executor/fixed-thread-executor threads)] @@ -360,17 +316,59 @@ (mapv (comp deref computations second) targets))))) -(defn- run-parallel! - "Runs the tasks for targets in multiple worker threads, chained by dependency - order. Returns a vector of result maps in the order the tasks finished executing." - [ctx threads targets] - (if (get-in ctx [:opts :output]) - ;; NOTE: this is done here rather than inside each task so that tasks - ;; starting across threads don't have a chance to see the `sh` var between - ;; rebindings. - (with-redefs [leiningen.core.eval/sh run-with-output] - (run-parallel* ctx threads targets)) - (run-parallel* ctx threads targets))) + +;; ## Task Entry + +(defn- select-projects + "Returns a vector of pairs of index numbers and symbols naming the selected + subprojects." + [monolith subprojects fprints opts] + (let [dependencies (dep/dependency-map subprojects) + targets (target/select monolith subprojects opts) + start (when-let [start (:start opts)] + (dep/resolve-name! (keys subprojects) (read-string start))) + marker (:changed opts)] + (-> + ;; Sort project names by dependency order. + (dep/topological-sort dependencies targets) + (cond->> + ;; Skip projects until the starting project, if provided. + start + (drop-while (partial not= start)) + ;; Skip projects whose fingerprint hasn't changed. + marker + (filter (partial fingerprint/changed? fprints marker))) + ;; Pair names up with an index [[i project-sym] ...] + (->> + (map-indexed vector))))) + + +(defn- print-report + "Reports information about the tasks given a results map." + [results elapsed] + (let [task-time (reduce + (keep :elapsed results)) + speedup (/ task-time elapsed)] + (lein/info (format "\n%s %11s" + (colorize [:bold :cyan] "Run time:") + (u/human-duration elapsed))) + (lein/info (format "%s %11s" + (colorize [:bold :cyan] "Task time:") + (u/human-duration task-time))) + (lein/info (format "%s %11.1f" + (colorize [:bold :cyan] "Speedup:") + speedup)) + (lein/info (->> results + (sort-by :elapsed) + (reverse) + (take 8) + (map #(format "%-45s %s %11s" + (colorize [:bold :yellow] (:name %)) + (if (:success %) " " "!") + (u/human-duration (:elapsed %)))) + (str/join "\n") + (str \newline + (colorize [:bold :cyan] "Slowest projects:") + \newline))))) (defn run-tasks @@ -379,8 +377,7 @@ (let [[monolith subprojects] (u/load-monolith! project) fprints (fingerprint/context monolith subprojects) opts (if-let [marker (:refresh opts)] - (-> opts - (assoc :changed marker)) + (assoc opts :changed marker) opts) targets (select-projects monolith subprojects fprints From 053f537cbdd221af85b9e586ee0762e6ea4a4459 Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 11:34:58 -0700 Subject: [PATCH 2/6] More 'each' refactoring to prep for silent option. --- src/lein_monolith/task/each.clj | 128 +++++++++++++++++++------------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index 3f0c4eb..67526e9 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -22,7 +22,8 @@ ClosingPipe Pipe RevivableInputStream) - java.io.OutputStream)) + java.io.OutputStream + java.time.Instant)) ;; ## Task Options @@ -77,6 +78,53 @@ +;; ## Task Initialization + +(def ^:private init-lock + "An object to lock on to ensure that projects are not initialized + concurrently. This prevents the mysterious 'unbound fn' errors that sometimes + crop up during parallel execution." + (Object.)) + + +(defn- resolve-tasks + "Perform an initial resolution of the task to prevent metadata-related + arglist errors when namespaces are loaded in parallel." + [project task+args] + (let [[task args] (lein/task-args task+args project)] + (lein/resolve-task task) + ;; Some tasks pull in other tasks, so also resolve them. + (condp = task + "do" + (doseq [subtask+args (lein-do/group-args args)] + (resolve-tasks project subtask+args)) + + "update-in" + (let [subtask+args (rest (drop-while #(not= "--" %) args))] + (resolve-tasks project subtask+args)) + + "with-profile" + (let [subtask+args (rest args)] + (resolve-tasks project subtask+args)) + + ;; default no-op + nil))) + + +(defn- apply-subproject-task + "Applies the task to the given subproject." + [subproject task] + (binding [lein/*exit-process?* false + eval/*dir* (:root subproject)] + (let [initialized (config/debug-profile "init-subproject" + (locking init-lock + (project/init-project + (plugin/add-middleware subproject))))] + (config/debug-profile "apply-task" + (lein/resolve-and-apply initialized task))))) + + + ;; ## Output Handling (defn- tee-output-stream @@ -106,7 +154,7 @@ (close [] - ; no-op + ;; no-op nil))) @@ -148,26 +196,6 @@ exit-value))))) -(def ^:private init-lock - "An object to lock on to ensure that projects are not initialized - concurrently. This prevents the mysterious 'unbound fn' errors that sometimes - crop up during parallel execution." - (Object.)) - - -(defn- apply-subproject-task - "Applies the task to the given subproject." - [subproject task] - (binding [lein/*exit-process?* false - eval/*dir* (:root subproject)] - (let [initialized (config/debug-profile "init-subproject" - (locking init-lock - (project/init-project - (plugin/add-middleware subproject))))] - (config/debug-profile "apply-task" - (lein/resolve-and-apply initialized task))))) - - (defn- apply-subproject-task-with-output "Applies the task to the given subproject, writing the task output to a file in the given directory." @@ -178,7 +206,7 @@ ; Write task header (.write file-output-stream (.getBytes (format "[%s] Applying task to %s/%s: %s\n\n" - (java.util.Date.) + (Instant/now) (:group subproject) (:name subproject) (str/join " " task)))) @@ -197,34 +225,10 @@ ; Write task footer (.write file-output-stream (.getBytes (format "\n[%s] Elapsed: %s\n" - (java.util.Date.) + (Instant/now) (u/human-duration (:elapsed @results)))))))))) -(defn- resolve-tasks - "Perform an initial resolution of the task to prevent metadata-related - arglist errors when namespaces are loaded in parallel." - [project task+args] - (let [[task args] (lein/task-args task+args project)] - (lein/resolve-task task) - ;; Some tasks pull in other tasks, so also resolve them. - (condp = task - "do" - (doseq [subtask+args (lein-do/group-args args)] - (resolve-tasks project subtask+args)) - - "update-in" - (let [subtask+args (rest (drop-while #(not= "--" %) args))] - (resolve-tasks project subtask+args)) - - "with-profile" - (let [subtask+args (rest args)] - (resolve-tasks project subtask+args)) - - ;; default no-op - nil))) - - (defn- run-task! "Runs the given task, returning a map of information about the run." [ctx target] @@ -316,8 +320,30 @@ (mapv (comp deref computations second) targets))))) +(defn- run-all* + "Run all tasks, using the `:parallel` option to determine whether to run them + serially or concurrently." + [ctx targets] + (if-let [threads (get-in ctx [:opts :parallel])] + (run-parallel! ctx (Integer/parseInt threads) targets) + (run-linear! ctx targets))) + + +(defn- run-all! + "Run all tasks, using the `:parallel` and `:output` options to determine + behavior." + [ctx targets] + (if (get-in ctx [:opts :output]) + ;; NOTE: this is done here rather than inside each task so that tasks + ;; starting across threads don't have a chance to see the `sh` var between + ;; rebindings. + (with-redefs [leiningen.core.eval/sh run-with-output] + (run-all* ctx targets)) + (run-all* ctx targets))) + + -;; ## Task Entry +;; ## Each Task (defn- select-projects "Returns a vector of pairs of index numbers and symbols naming the selected @@ -398,9 +424,7 @@ :num-targets n :task task :opts opts} - results (if-let [threads (:parallel opts)] - (run-parallel! ctx (Integer/parseInt threads) targets) - (run-linear! ctx targets)) + results (run-all! ctx targets) elapsed (/ (- (System/nanoTime) start-time) 1000000.0)] (when (:report opts) (print-report results elapsed)) From 5a880d276f2f52b85fedc2ed81ec4a1a852a25c4 Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 12:01:13 -0700 Subject: [PATCH 3/6] Start wiring in silent option. --- src/lein_monolith/task/each.clj | 56 +++++++++++++++++++++------------ src/leiningen/monolith.clj | 1 + 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index 67526e9..5fde3c1 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -34,6 +34,7 @@ {:parallel 1 :endure 0 :report 0 + :silent 0 :output 1 :upstream 0 :downstream 0 @@ -53,6 +54,8 @@ [:endure]) (when (:report opts) [:report]) + (when (:silent opts) + [:silent]) (when-let [out-dir (:output opts)] [:output out-dir]) (when-let [in (seq (:in opts))] @@ -78,7 +81,7 @@ -;; ## Task Initialization +;; ## Project Initialization (def ^:private init-lock "An object to lock on to ensure that projects are not initialized @@ -87,6 +90,15 @@ (Object.)) +(defn- init-project + "Initialize the given subproject to prepare to run a task in it." + [subproject] + (locking init-lock + (config/debug-profile "init-subproject" + (project/init-project + (plugin/add-middleware subproject))))) + + (defn- resolve-tasks "Perform an initial resolution of the task to prevent metadata-related arglist errors when namespaces are loaded in parallel." @@ -94,7 +106,7 @@ (let [[task args] (lein/task-args task+args project)] (lein/resolve-task task) ;; Some tasks pull in other tasks, so also resolve them. - (condp = task + (case task "do" (doseq [subtask+args (lein-do/group-args args)] (resolve-tasks project subtask+args)) @@ -111,21 +123,11 @@ nil))) -(defn- apply-subproject-task - "Applies the task to the given subproject." - [subproject task] - (binding [lein/*exit-process?* false - eval/*dir* (:root subproject)] - (let [initialized (config/debug-profile "init-subproject" - (locking init-lock - (project/init-project - (plugin/add-middleware subproject))))] - (config/debug-profile "apply-task" - (lein/resolve-and-apply initialized task))))) +;; ## Output Handling +(declare apply-subproject-task) -;; ## Output Handling (defn- tee-output-stream "Constructs a proxy of an OutputStream that will write a copy of the bytes @@ -203,7 +205,7 @@ (let [out-file (io/file out-dir (:group subproject) (str (:name subproject) ".txt"))] (io/make-parents out-file) (with-open [file-output-stream (io/output-stream out-file :append true)] - ; Write task header + ;; Write task header (.write file-output-stream (.getBytes (format "[%s] Applying task to %s/%s: %s\n\n" (Instant/now) @@ -211,7 +213,7 @@ (:name subproject) (str/join " " task)))) (try - ; Run task with output capturing. + ;; Run task with output capturing. (binding [*task-file-output* file-output-stream] (apply-subproject-task subproject task)) (catch Exception ex @@ -222,13 +224,26 @@ (cst/print-cause-trace ex))))) (throw ex)) (finally - ; Write task footer + ;; Write task footer (.write file-output-stream (.getBytes (format "\n[%s] Elapsed: %s\n" (Instant/now) (u/human-duration (:elapsed @results)))))))))) + +;; ## Task Execution + +(defn- apply-subproject-task + "Applies the task to the given subproject." + [subproject task] + (binding [lein/*exit-process?* false + eval/*dir* (:root subproject)] + (let [initialized (init-project subproject)] + (config/debug-profile "apply-task" + (lein/resolve-and-apply initialized task))))) + + (defn- run-task! "Runs the given task, returning a map of information about the run." [ctx target] @@ -330,10 +345,11 @@ (defn- run-all! - "Run all tasks, using the `:parallel` and `:output` options to determine - behavior." + "Run all tasks, using the `:parallel`, `:silent`, and `:output` options to + determine behavior." [ctx targets] - (if (get-in ctx [:opts :output]) + (if (or (get-in ctx [:opts :silent]) + (get-in ctx [:opts :output])) ;; NOTE: this is done here rather than inside each task so that tasks ;; starting across threads don't have a chance to see the `sh` var between ;; rebindings. diff --git a/src/leiningen/monolith.clj b/src/leiningen/monolith.clj index 3ed8a22..fdcfb4f 100644 --- a/src/leiningen/monolith.clj +++ b/src/leiningen/monolith.clj @@ -149,6 +149,7 @@ :parallel Run tasks in parallel across a fixed thread pool. :endure Continue executing the task even if some subprojects fail. :report Print a detailed timing report after running tasks. + :silent Don't print project output unless the task fails. :output Save each project's individual output in the given directory. Targeting Options: From 7ec7e603805cc6e64f017d212d19ef7630a64f31 Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 16:16:08 -0700 Subject: [PATCH 4/6] Simplify task results, first pass at silent implementation. --- src/lein_monolith/task/each.clj | 160 +++++++++++++++++++------------- src/lein_monolith/task/util.clj | 8 ++ 2 files changed, 104 insertions(+), 64 deletions(-) diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index 5fde3c1..a642888 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -22,7 +22,9 @@ ClosingPipe Pipe RevivableInputStream) - java.io.OutputStream + (java.io + ByteArrayOutputStream + OutputStream) java.time.Instant)) @@ -126,7 +128,21 @@ ;; ## Output Handling -(declare apply-subproject-task) +(def ^:private output-lock + "An object to lock on to ensure that project output is not interleaved when + running in silent mode." + (Object.)) + + +(def ^:private ^:dynamic *task-capture-output* + "If bound, write task output to this stream *instead* of the standard output + and error streams." + nil) + + +(def ^:private ^:dynamic *task-file-output* + "If bound, copy task output to this stream to record it to a log file." + nil) (defn- tee-output-stream @@ -160,18 +176,12 @@ nil))) -(def ^:dynamic *task-file-output* nil) - - (defn- run-with-output "A version of `leiningen.core.eval/sh` that streams in/out/err, teeing output to the given file." [& cmd] (when eval/*pump-in* (rebind-io!)) - (when-not *task-file-output* - (throw (IllegalStateException. - (str "Cannot run task without bound *task-file-output*: " (pr-str cmd))))) (let [cmd (into-array String cmd) env (into-array String (@#'eval/overridden-env eval/*env*)) proc (.exec (Runtime/getRuntime) @@ -183,8 +193,16 @@ (with-open [out (.getInputStream proc) err (.getErrorStream proc) in (.getOutputStream proc)] - (let [pump-out (doto (Pipe. out (tee-output-stream System/out *task-file-output*)) .start) - pump-err (doto (Pipe. err (tee-output-stream System/err *task-file-output*)) .start) + (let [out-dest (cond-> (or *task-capture-output* System/out) + *task-file-output* + (tee-output-stream *task-file-output*)) + err-dest (cond-> (or *task-capture-output* System/err) + *task-file-output* + (tee-output-stream *task-file-output*)) + pump-out (doto (Pipe. out ^OutputStream out-dest) + (.start)) + pump-err (doto (Pipe. err ^OutputStream err-dest) + (.start)) pump-in (ClosingPipe. System/in in)] (when eval/*pump-in* (.start pump-in)) (.join pump-out) @@ -198,11 +216,12 @@ exit-value))))) -(defn- apply-subproject-task-with-output - "Applies the task to the given subproject, writing the task output to a file - in the given directory." - [subproject task out-dir results] - (let [out-file (io/file out-dir (:group subproject) (str (:name subproject) ".txt"))] +(defn- apply-with-output + "Applies the function to the given subproject, writing the task output to a + file in the given directory." + [out-dir f subproject task] + (let [out-file (io/file out-dir (:group subproject) (str (:name subproject) ".txt")) + elapsed (u/stopwatch)] (io/make-parents out-file) (with-open [file-output-stream (io/output-stream out-file :append true)] ;; Write task header @@ -215,7 +234,7 @@ (try ;; Run task with output capturing. (binding [*task-file-output* file-output-stream] - (apply-subproject-task subproject task)) + (f subproject task)) (catch Exception ex (.write file-output-stream (.getBytes (format "\nERROR: %s\n%s" @@ -228,7 +247,7 @@ (.write file-output-stream (.getBytes (format "\n[%s] Elapsed: %s\n" (Instant/now) - (u/human-duration (:elapsed @results)))))))))) + (u/human-duration @elapsed))))))))) @@ -249,31 +268,41 @@ [ctx target] ;; Try to reclaim some memory before running the task. (System/gc) - (let [start (System/nanoTime) + (let [subproject (get-in ctx [:subprojects target]) opts (:opts ctx) - subproject (get-in ctx [:subprojects target]) - results (delay {:name target - :elapsed (/ (- (System/nanoTime) start) 1000000.0)}) marker (:changed opts) - fprints (:fingerprints ctx)] + fprints (:fingerprints ctx) + elapsed (u/stopwatch) + task-output (when (:silent opts) + (ByteArrayOutputStream.))] (try (lein/info (format "\nApplying to %s%s" (colorize [:bold :yellow] target) (if marker (str " (" (fingerprint/explain-str fprints marker target) ")") ""))) - (if-let [out-dir (get-in ctx [:opts :output])] - ;; Capture output to file. - (apply-subproject-task-with-output subproject (:task ctx) out-dir results) - ;; Run without output capturing. - (apply-subproject-task subproject (:task ctx))) + ;; Bind appropriate output options and apply the task. + (binding [*task-capture-output* task-output] + (if-let [out-dir (get-in ctx [:opts :output])] + (apply-with-output out-dir apply-subproject-task subproject (:task ctx)) + (apply-subproject-task subproject (:task ctx)))) + ;; Save updated fingerprint if refreshing. (when (:refresh opts) (fingerprint/save! fprints marker target) (lein/info (format "Saved %s fingerprint for %s" (colorize :bold marker) (colorize [:bold :yellow] target)))) - (assoc @results :success true) + ;; Return successful task result. + {:name target + :elapsed @elapsed + :success true} (catch Exception ex + ;; When silent, grab output lock and print task output. + (when (:silent opts) + (locking output-lock + (print (str task-output)) + (flush))) + ;; Print convenience resume tip for user. (when-not (or (:parallel opts) (:endure opts)) (let [resume-args (into ["lein" "monolith" "each"] @@ -285,6 +314,7 @@ (lein/warn (format "\n%s %s\n" (colorize [:bold :red] "Resume with:") (str/join " " resume-args))))) + ;; Fail or continue depending on whether endure is enabled. (if (:endure opts) (lein/warn (format "\n%s: %s\n%s" (colorize [:bold :red] "ERROR") @@ -292,13 +322,16 @@ (with-out-str (cst/print-cause-trace ex)))) (throw ex)) - (assoc @results :success false, :error ex)) + {:name target + :elapsed @elapsed + :success false + :error ex}) (finally (lein/info (format "Completed %s (%s/%s) in %s" (colorize [:bold :yellow] target) (colorize :cyan (swap! (:completions ctx) inc)) (colorize :cyan (:num-targets ctx)) - (colorize [:bold :cyan] (u/human-duration (:elapsed @results))))))))) + (colorize [:bold :cyan] (u/human-duration @elapsed)))))))) (defn- run-linear! @@ -423,37 +456,36 @@ opts) targets (select-projects monolith subprojects fprints - (u/globalize-opts project opts)) - n (inc (or (first (last targets)) -1)) - start-time (System/nanoTime)] - (if (empty? targets) - (lein/info "Target selection matched zero subprojects; nothing to do") - (do - (lein/info "Applying" - (colorize [:bold :cyan] (str/join " " task)) - "to" (colorize :cyan (count targets)) - "subprojects...") - (let [ctx {:monolith monolith - :subprojects subprojects - :fingerprints fprints - :completions (atom (ffirst targets)) - :num-targets n - :task task - :opts opts} - results (run-all! ctx targets) - elapsed (/ (- (System/nanoTime) start-time) 1000000.0)] - (when (:report opts) - (print-report results elapsed)) - (if-let [failures (seq (map :name (remove :success results)))] - (lein/abort (format "\n%s: Applied %s to %s projects in %s with %d failures: %s" - (colorize [:bold :red] "FAILURE") - (colorize [:bold :cyan] (str/join " " task)) - (colorize :cyan (count targets)) - (u/human-duration elapsed) - (count failures) - (str/join " " failures))) - (lein/info (format "\n%s: Applied %s to %s projects in %s" - (colorize [:bold :green] "SUCCESS") - (colorize [:bold :cyan] (str/join " " task)) - (colorize :cyan (count targets)) - (u/human-duration elapsed))))))))) + (u/globalize-opts project opts))] + (if (seq targets) + (lein/info "Applying" + (colorize [:bold :cyan] (str/join " " task)) + "to" (colorize :cyan (count targets)) + "subprojects...") + (lein/info "Target selection matched zero subprojects; nothing to do")) + (when (seq targets) + (let [elapsed (u/stopwatch) + results (run-all! + {:monolith monolith + :subprojects subprojects + :fingerprints fprints + :completions (atom (ffirst targets)) + :num-targets (inc (or (first (last targets)) -1)) + :task task + :opts opts} + targets)] + (when (:report opts) + (print-report results @elapsed)) + (if-let [failures (seq (map :name (remove :success results)))] + (lein/abort (format "\n%s: Applied %s to %s projects in %s with %d failures: %s" + (colorize [:bold :red] "FAILURE") + (colorize [:bold :cyan] (str/join " " task)) + (colorize :cyan (count targets)) + (u/human-duration @elapsed) + (count failures) + (str/join " " failures))) + (lein/info (format "\n%s: Applied %s to %s projects in %s" + (colorize [:bold :green] "SUCCESS") + (colorize [:bold :cyan] (str/join " " task)) + (colorize :cyan (count targets)) + (u/human-duration @elapsed)))))))) diff --git a/src/lein_monolith/task/util.clj b/src/lein_monolith/task/util.clj index 95abe53..eed57b3 100644 --- a/src/lein_monolith/task/util.clj +++ b/src/lein_monolith/task/util.clj @@ -79,6 +79,14 @@ (:downstream opts) (update :downstream-of conj (:name project))))) +(defn stopwatch + "Construct a timer which will contain the number of milliseconds elapsed + between its creation and when it is dereferenced." + [] + (let [start (System/nanoTime)] + (delay (/ (- (System/nanoTime) start) 1e6)))) + + (defn human-duration "Renders a duration in milliseconds in hour:minute:second.ms format." [duration] From 06d9e5ec61a7e5d6506987150be99fa165f40a60 Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 16:28:42 -0700 Subject: [PATCH 5/6] Fix NPE when not capturing output. --- src/lein_monolith/task/each.clj | 3 ++- src/leiningen/monolith.clj | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index a642888..7af9e23 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -212,7 +212,8 @@ (.kill ^RevivableInputStream System/in) (.join pump-in) (.resurrect ^RevivableInputStream System/in)) - (.flush ^OutputStream *task-file-output*) + (when *task-file-output* + (.flush ^OutputStream *task-file-output*)) exit-value))))) diff --git a/src/leiningen/monolith.clj b/src/leiningen/monolith.clj index fdcfb4f..e23517f 100644 --- a/src/leiningen/monolith.clj +++ b/src/leiningen/monolith.clj @@ -146,11 +146,11 @@ project to resume from. General Options: - :parallel Run tasks in parallel across a fixed thread pool. - :endure Continue executing the task even if some subprojects fail. - :report Print a detailed timing report after running tasks. - :silent Don't print project output unless the task fails. - :output Save each project's individual output in the given directory. + :parallel Run tasks in parallel across a fixed thread pool. + :endure Continue executing the task even if some subprojects fail. + :report Print a detailed timing report after running tasks. + :silent Don't print task output unless a subproject fails. + :output Save each project's individual output in the given directory. Targeting Options: :in Add the named projects directly to the targets. From aba622775c6992e2167265e1925dc21e6349be2d Mon Sep 17 00:00:00 2001 From: Greg Look Date: Wed, 30 Sep 2020 16:36:35 -0700 Subject: [PATCH 6/6] Update README and CHANGELOG. --- CHANGELOG.md | 7 +++++++ README.md | 1 + 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e94761f..cfdef62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ This project adheres to [Semantic Versioning](http://semver.org/). [#36](https://github.com/amperity/lein-monolith/issues/36) [#74](https://github.com/amperity/lein-monolith/pull/74) +### Added +- The `each` task supports a new `:silent` option, which will suppress task + output for successful projects. This can be useful in large CI builds where + the output is only consulted in the event of failure. + [#37](https://github.com/amperity/lein-monolith/issues/37) + [#81](https://github.com/amperity/lein-monolith/pull/81) + ## [1.5.0] - 2020-09-17 diff --git a/README.md b/README.md index 0ac26ec..491825e 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ In addition to targeting options, `each` accepts: is useful in situations such as CI tests, where you don't want a failure to halt iteration. - `:report` show a detailed timing report after the tasks finish executing. +- `:silent` suppress task output for successful projects. - `:output` path to a directory to save individual build output in. #### Incremental Builds