Deprecate use of ec['parallel'] and fix updating the template value

easybuild/framework/easyblock.py

@@ -1861,7 +1861,7 @@ def skip_extensions_parallel(self, exts_filter):
                     self.update_exts_progress_bar(exts_pbar_label)
 
             # start additional checks asynchronously
-            while exts_queue and len(running_checks_ids) < self.cfg['parallel']:
+            while exts_queue and len(running_checks_ids) < self.cfg.parallel:
                 idx, ext = exts_queue.pop(0)
                 cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
                 async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin,
@@ -2024,7 +2024,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
 
             for _ in range(max_iter):
 
-                if not (exts_queue and len(running_exts) < self.cfg['parallel']):
+                if not (exts_queue and len(running_exts) < self.cfg.parallel):
                     break
 
                 # check whether extension at top of the queue is ready to install
@@ -2248,19 +2248,22 @@ def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
         # set level of parallelism for build
         par = build_option('parallel')
-        cfg_par = self.cfg['parallel']
-        if cfg_par is None:
+        if par is not None:
             self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
-        elif par is None:
-            par = cfg_par
-            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-        else:
-            par = min(int(par), int(cfg_par))
-            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
+
+        # Transitional only in case some easyblocks still set/change cfg['parallel']
+        # Use _parallelLegacy to avoid deprecation warnings
+        # Remove for EasyBuild 5.0
+        cfg_par = self.cfg['_parallelLegacy']
+        if cfg_par is not None:
+            if par is None:
+                par = cfg_par
+            else:
+                par = min(int(par), int(cfg_par))
 
         par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
         self.log.info("Setting parallelism: %s" % par)
-        self.cfg['parallel'] = par
+        self.cfg.parallel = par
 
     def remove_module_file(self):
         """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""

easybuild/framework/easyconfig/default.py

@@ -110,8 +110,6 @@
     'installopts': ['', 'Extra options for installation', BUILD],
     'maxparallel': [None, 'Max degree of parallelism', BUILD],
     'module_only': [False, 'Only generate module file', BUILD],
-    'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
-                        'cores, active cpuset and restrictions in ulimit)'), BUILD],
     'patches': [[], "List of patches to apply", BUILD],
     'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
     'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],

easybuild/framework/easyconfig/easyconfig.py

@@ -118,11 +118,16 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
     def new_ec_method(self, key, *args, **kwargs):
         """Check whether any replace easyconfig parameters are still used"""
         # map deprecated parameters to their replacements, issue deprecation warning(/error)
-        if key in DEPRECATED_PARAMETERS:
+        if key == 'parallel':
+            _log.deprecated("Easyconfig parameter 'parallel' is deprecated, "
+                            "use 'maxparallel' or the parallel property instead.", '5.0')
+            # Use a "hidden" property for now to match behavior as closely as possible
+            key = '_parallelLegacy'
+        elif key in DEPRECATED_PARAMETERS:
             depr_key = key
             key, ver = DEPRECATED_PARAMETERS[depr_key]
             _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver)
-        if key in REPLACED_PARAMETERS:
+        elif key in REPLACED_PARAMETERS:
             _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
         return ec_method(self, key, *args, **kwargs)
 
@@ -140,7 +145,9 @@ def is_local_var_name(name):
     """
     res = False
     if name.startswith(LOCAL_VAR_PREFIX) or name.startswith('_'):
-        res = True
+        # Remove with EasyBuild 5.0
+        if name != '_parallelLegacy':
+            res = True
     # __builtins__ is always defined as a 'local' variables
     # single-letter local variable names are allowed (mainly for use in list comprehensions)
     # in Python 2, variables defined in list comprehensions leak to the outside (no longer the case in Python 3)
@@ -511,6 +518,11 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         self.iterate_options = []
         self.iterating = False
 
+        # Storage for parallel property. Mark as unset initially
+        self._parallel = None
+        # Legacy value, remove with EasyBuild 5.0
+        self._config['_parallelLegacy'] = [None, '', ('', )]
+
         # parse easyconfig file
         self.build_specs = build_specs
         self.parse()
@@ -713,6 +725,11 @@ def parse(self):
         if missing_mandatory_keys:
             raise EasyBuildError("mandatory parameters not provided in %s: %s", self.path, missing_mandatory_keys)
 
+        if 'parallel' in ec_vars:
+            # Replace value and issue better warning for EC params (as opposed to warnings meant for easyblocks)
+            self.log.deprecated("Easyconfig parameter 'parallel' is deprecated, use 'maxparallel' instead.", '5.0')
+            ec_vars['_parallelLegacy'] = ec_vars.pop('parallel')
+
         # provide suggestions for typos. Local variable names are excluded from this check
         possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                           for key in ec_vars if not is_local_var_name(key) and key not in self]
@@ -1219,6 +1236,22 @@ def all_dependencies(self):
 
         return self._all_dependencies
 
+    @property
+    def parallel(self):
+        """Number of parallel jobs to be used for building etc."""
+        if self._parallel is None:
+            raise EasyBuildError("Parallelism in EasyConfig not set yet. "
+                                 "Need to call the easyblocks set_parallel first.")
+        return self._parallel
+
+    @parallel.setter
+    def parallel(self, value):
+        # Update backstorage and template value
+        self._parallel = value
+        self.template_values['parallel'] = value
+        # Backwards compat only. Remove with EasyBuild 5.0
+        self._config['_parallelLegacy'][0] = value
+
     def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False):
         """
         Dump this easyconfig to file, with the given filename.

easybuild/framework/easyconfig/format/format.py

@@ -69,7 +69,7 @@
     ['preconfigopts', 'configopts'],
     ['prebuildopts', 'buildopts'],
     ['preinstallopts', 'installopts'],
-    ['parallel', 'maxparallel'],
+    ['maxparallel'],
 ]
 LAST_PARAMS = ['exts_default_options', 'exts_list',
                'sanity_check_paths', 'sanity_check_commands',

easybuild/framework/easyconfig/templates.py

@@ -60,7 +60,6 @@
     'bitbucket_account',
     'github_account',
     'name',
-    'parallel',
     'version',
     'versionsuffix',
     'versionprefix',
@@ -100,6 +99,7 @@
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
     ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
     ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
+    ('parallel', "Degree of parallelism for e.g. make"),
 ]
 
 # constant templates that can be used in easyconfigs

easybuild/scripts/mk_tmpl_easyblock_for.py

@@ -165,7 +165,7 @@ def build_step(self):
         comp_fam = comp_map[self.toolchain.comp_family()]
 
         # enable parallel build
-        par = self.cfg['parallel']
+        par = self.cfg.parallel
         cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam)
         run_cmd(cmd, log_all=True, simple=True, log_ok=True)
 

easybuild/tools/systemtools.py

@@ -1212,6 +1212,8 @@ def get_default_parallelism():
             raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
 
     if maxpar is not None and maxpar < par:
+        if maxpar is False:
+            maxpar = 1
         _log.info("Limiting parallelism from %s to %s", par, maxpar)
         par = maxpar
 

test/framework/easyblock.py

@@ -2116,46 +2116,119 @@ def test_parallel(self):
         os.close(handle)
         write_file(toy_ec3, toytxt + "\nparallel = False")
 
+        handle, toy_ec4 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
+        os.close(handle)
+        write_file(toy_ec4, toytxt + "\nmaxparallel = 67")
+
+        handle, toy_ec5 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
+        os.close(handle)
+        write_file(toy_ec5, toytxt + "\nmaxparallel = False")
+
+        import easybuild.tools.systemtools as st
+        auto_parallel = 1337
+        st.det_parallelism._default_parallelism = auto_parallel
+
         # default: parallelism is derived from # available cores + ulimit
         test_eb = EasyBlock(EasyConfig(toy_ec))
         test_eb.check_readiness_step()
-        self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0)
+        self.assertEqual(test_eb.cfg.parallel, auto_parallel)
 
         # only 'parallel' easyconfig parameter specified (no 'parallel' build option)
-        test_eb = EasyBlock(EasyConfig(toy_ec1))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec1))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 123)
+        self.assertEqual(test_eb.cfg.parallel, 123)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 123)
 
         # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
-        test_eb = EasyBlock(EasyConfig(toy_ec2))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec2))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 67)
+        self.assertEqual(test_eb.cfg.parallel, 67)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 67)
 
         # make sure 'parallel = False' is not overriden (no 'parallel' build option)
-        test_eb = EasyBlock(EasyConfig(toy_ec3))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec3))
+        test_eb.check_readiness_step()
+        self.assertEqual(test_eb.cfg.parallel, False)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], False)
+
+        # only 'maxparallel' easyconfig parameter specified (no 'parallel' build option)
+        test_eb = EasyBlock(EasyConfig(toy_ec4))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], False)
+        self.assertEqual(test_eb.cfg.parallel, 67)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 67)
+
+        # make sure 'maxparallel = False' is treated as 1 (no 'parallel' build option)
+        test_eb = EasyBlock(EasyConfig(toy_ec5))
+        test_eb.check_readiness_step()
+        self.assertEqual(test_eb.cfg.parallel, 1)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 1)
 
         # only 'parallel' build option specified
         init_config(build_options={'parallel': '97', 'validate': False})
         test_eb = EasyBlock(EasyConfig(toy_ec))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 97)
+        self.assertEqual(test_eb.cfg.parallel, 97)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 97)
 
         # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
-        test_eb = EasyBlock(EasyConfig(toy_ec1))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec1))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 97)
+        self.assertEqual(test_eb.cfg.parallel, 97)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 97)
 
         # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
-        test_eb = EasyBlock(EasyConfig(toy_ec2))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec2))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 67)
+        self.assertEqual(test_eb.cfg.parallel, 67)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 67)
 
         # make sure 'parallel = False' is not overriden (with 'parallel' build option)
-        test_eb = EasyBlock(EasyConfig(toy_ec3))
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            test_eb = EasyBlock(EasyConfig(toy_ec3))
+        test_eb.check_readiness_step()
+        self.assertEqual(test_eb.cfg.parallel, 0)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 0)
+
+        # only 'maxparallel' easyconfig parameter specified (with 'parallel' build option)
+        test_eb = EasyBlock(EasyConfig(toy_ec4))
+        test_eb.check_readiness_step()
+        self.assertEqual(test_eb.cfg.parallel, 67)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 67)
+
+        # make sure 'maxparallel = False' is treated as 1 (with 'parallel' build option)
+        test_eb = EasyBlock(EasyConfig(toy_ec5))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 0)
+        self.assertEqual(test_eb.cfg.parallel, 1)
+        with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+            self.assertEqual(test_eb.cfg['parallel'], 1)
+
+        # Template updated correctly
+        test_eb.cfg['buildopts'] = '-j %(parallel)s'
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 1')
+        # Might be done in an easyblock step
+        test_eb.cfg.parallel = 42
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 42')
+        # Unaffected by build settings
+        test_eb.cfg.parallel = 421337
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 421337')
+
+        # Reset mocked value
+        del st.det_parallelism._default_parallelism
 
     def test_guess_start_dir(self):
         """Test guessing the start dir."""

test/framework/easyconfig.py

@@ -656,7 +656,7 @@ def test_tweaking(self):
             'version = "3.14"',
             'toolchain = {"name": "GCC", "version": "4.6.3"}',
             'patches = %s',
-            'parallel = 1',
+            'maxparallel = 1',
             'keepsymlinks = True',
         ]) % str(patches)
         self.prep()
@@ -679,7 +679,7 @@ def test_tweaking(self):
             # It should be possible to overwrite values with True/False/None as they often have special meaning
             'runtest': 'False',
             'hidden': 'True',
-            'parallel': 'None',  # Good example: parallel=None means "Auto detect"
+            'maxparallel': 'None',  # Good example: maxparallel=None means "unlimitted"
             # Adding new options (added only by easyblock) should also be possible
             # and in case the string "True/False/None" is really wanted it is possible to quote it first
             'test_none': '"False"',
@@ -696,7 +696,7 @@ def test_tweaking(self):
         self.assertEqual(eb['patches'], new_patches)
         self.assertIs(eb['runtest'], False)
         self.assertIs(eb['hidden'], True)
-        self.assertIsNone(eb['parallel'])
+        self.assertIsNone(eb['maxparallel'])
         self.assertEqual(eb['test_none'], 'False')
         self.assertEqual(eb['test_bool'], 'True')
         self.assertEqual(eb['test_123'], 'None')
@@ -1774,8 +1774,7 @@ def foo(key):
 
     def test_deprecated_easyconfig_parameters(self):
         """Test handling of replaced easyconfig parameters."""
-        os.environ.pop('EASYBUILD_DEPRECATED')
-        easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version
+        self.allow_deprecated_behaviour()
         init_config()
 
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
@@ -3336,7 +3335,6 @@ def test_template_constant_dict(self):
             'namelower': 'gzip',
             'nameletter': 'g',
             'nameletterlower': 'g',
-            'parallel': None,
             'sysroot': '',
             'toolchain_name': 'foss',
             'toolchain_version': '2018a',
@@ -3372,7 +3370,6 @@ def test_template_constant_dict(self):
 
         res = template_constant_dict(ec)
         res.pop('arch')
-        expected['parallel'] = 42
         self.assertEqual(res, expected)
 
         toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb')
@@ -3414,7 +3411,6 @@ def test_template_constant_dict(self):
             'toolchain_name': 'system',
             'toolchain_version': 'system',
             'nameletterlower': 't',
-            'parallel': None,
             'pymajver': '3',
             'pyminver': '7',
             'pyshortver': '3.7',
@@ -3448,7 +3444,7 @@ def test_template_constant_dict(self):
         ec = EasyConfigParser(filename=test_ec).get_config_dict()
 
         expected['module_name'] = None
-        for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'):
+        for key in ('bitbucket_account', 'github_account', 'versionprefix'):
             del expected[key]
 
         dep_names = [x[0] for x in ec['dependencies']]

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb

@@ -31,7 +31,7 @@ dependencies = [
     # ('Tcl', '8.6.4'),
 ]
 
-parallel = 1
+maxparallel = 1
 
 sanity_check_paths = {
     'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],