From 387f89fa7aa5df35401e100648c9278d433c81ba Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Mon, 13 Feb 2023 11:53:27 -0800 Subject: [PATCH] Increase the default value of rule_threads_max to avoid deadlocks when using tar file inputs. --- src/python/pants/option/global_options.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 705ff83ff061..3197a9b1a589 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -1055,7 +1055,11 @@ class BootstrapOptions: help=softwrap( """ The maximum number of threads to use to execute `@rule` logic. Defaults to - a small multiple of `--rule-threads-core`. + `16 * --rule-threads-core`. + + Note that setting too low a `--rule-threads-max` value can lead to deadlocks: Pants + uses blocking operations internally for a few use cases, and if the pool of blocking + threads is exhausted, those use cases will wait. """ ), ) @@ -1854,10 +1858,22 @@ def validate_remote_headers(opt_name: str) -> None: @staticmethod def create_py_executor(bootstrap_options: OptionValueContainer) -> PyExecutor: + # NB: See the `--rule-threads-max` option help for a warning on setting this too low. + # + # This value is chosen somewhat arbitrarily, but has a few concerns at play: + # * When set too low, tasks using `Executor::spawn_blocking` on the Rust side can deadlock + # when too many blocking operations already running. + # * Higher thread counts mean less bounded access to the LMDB store (which is the primary user + # of blocking tasks), which is good up to a point, but then begins to increase kernel time + # as many threads are blocked waiting for IO and locks. + # + # The value 16 was chosen to avoid deadlocks with all current `spawn_blocking` calls, and based + # on the observation that performance drops off by 2-3% points (on my machine!) when multiples + # of 32 and 64. rule_threads_max = ( bootstrap_options.rule_threads_max if bootstrap_options.rule_threads_max - else 4 * bootstrap_options.rule_threads_core + else 16 * bootstrap_options.rule_threads_core ) return PyExecutor( core_threads=bootstrap_options.rule_threads_core, max_threads=rule_threads_max