From de7015feef729d54f33cfe077c404787d48375c1 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 3 Feb 2021 12:25:45 +0000 Subject: [PATCH] module loader: Utilize CPU better --- internal/terraform/module/module_loader.go | 33 ++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/internal/terraform/module/module_loader.go b/internal/terraform/module/module_loader.go index 46fada29..6219d10a 100644 --- a/internal/terraform/module/module_loader.go +++ b/internal/terraform/module/module_loader.go @@ -22,15 +22,13 @@ type moduleLoader struct { } func newModuleLoader() *moduleLoader { - nonPrioParallelism := 2 * runtime.NumCPU() - prioParallelism := 1 * runtime.NumCPU() - + p := loaderParallelism(runtime.NumCPU()) plc, lc := int64(0), int64(0) ml := &moduleLoader{ queue: newModuleOpsQueue(), logger: defaultLogger, - nonPrioParallelism: int64(nonPrioParallelism), - prioParallelism: int64(prioParallelism), + nonPrioParallelism: p.NonPriority, + prioParallelism: p.Priority, opsToDispatch: make(chan ModuleOperation, 1), loadingCount: &lc, prioLoadingCount: &plc, @@ -39,6 +37,31 @@ func newModuleLoader() *moduleLoader { return ml } +type parallelism struct { + NonPriority, Priority int64 +} + +func loaderParallelism(cpu int) parallelism { + // Cap utilization for powerful machines + if cpu >= 4 { + return parallelism{ + NonPriority: int64(3), + Priority: int64(1), + } + } + if cpu == 3 { + return parallelism{ + NonPriority: int64(2), + Priority: int64(1), + } + } + + return parallelism{ + NonPriority: 1, + Priority: 1, + } +} + func (ml *moduleLoader) SetLogger(logger *log.Logger) { ml.logger = logger }