diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index c6a60acc1d..be9bb28bd3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,9 +90,6 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), - ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specify by the --sysroot configuration option"), - ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), @@ -102,6 +99,10 @@ ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), + ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"), + ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specified by the --sysroot configuration option"), ] # constant templates that can be used in easyconfigs @@ -210,6 +211,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback template_values['sysroot'] = build_option('sysroot') or '' + # set 'software_commit' template based on 'software_commit' configuration option, default to None + template_values['software_commit'] = build_option('software_commit') or '' + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 44362e6858..84d76dc5f4 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -259,6 +259,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_override_dirs', 'required_linked_shared_libs', 'skip', + 'software_commit', 'stop', 'subdir_user_modules', 'sysroot', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5a1cf689a4..49c11cfe83 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -515,6 +515,9 @@ def override_options(self): None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), + 'software-commit': ( + "Git commit to use for the target software build (robot capabilities are automatically disabled)", + None, 'store', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), @@ -1103,6 +1106,12 @@ def _postprocess_checks(self): else: raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot) + # 'software_commit' is specific to a particular software package and cannot be used with 'robot' + if self.options.software_commit: + if self.options.robot is not None: + print_warning("To allow use of --software-commit robot resolution is being disabled") + self.options.robot = None + self.log.info("Checks on configuration options passed") def get_cfg_opt_abs_path(self, opt_name, path): diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 92bd58e97d..c43c84be0e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1451,6 +1451,35 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_software_commit_template(self): + """Test the %(software_commit)s template""" + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"' + write_file(test_ec, test_ec_txt) + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=") + self.assertEqual(ec['buildopts'], "--some-opt=") + self.assertEqual(ec['installopts'], "--some-opt=") + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + software_commit = '1234bc' + update_build_option('software_commit', software_commit) + + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -3345,6 +3374,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'software_commit': '', 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', @@ -3427,6 +3457,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'software_commit': '', 'sysroot': '', 'version': '0.01', 'version_major': '0', @@ -3492,6 +3523,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'software_commit': '', 'sysroot': '', 'version': '1.2.3', 'version_major': '1', diff --git a/test/framework/options.py b/test/framework/options.py index 056d77ed6b..04b622d3dc 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -612,6 +612,8 @@ def run_test(fmt=None): r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + r'^``%\(software_commit\)s``\s+Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] else: pattern_lines = [ @@ -624,6 +626,8 @@ def run_test(fmt=None): r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + r'^\s+%\(software_commit\)s: Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] for pattern_line in pattern_lines: @@ -6834,6 +6838,23 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + def test_software_commit(self): + """Test use of --software-commit option.""" + + software_commit = "23be34" + software_commit_arg = '--software-commit=' + software_commit + # Add robot to also test that it gets disabled + stdout, stderr = self._run_mock_eb([software_commit_arg, '--show-config', '--robot'], raise_error=True) + + warning_regex = re.compile(r'.*WARNING:.*--software-commit robot resolution is being disabled.*', re.M) + software_commit_regex = re.compile(r'^software-commit\s*\(C\) = %s$' % software_commit, re.M) + robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M) + + self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr)) + self.assertTrue(software_commit_regex.search(stdout), + "Pattern '%s' not found in: %s" % (software_commit_regex, stdout)) + self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout)) + def test_accept_eula_for(self): """Test --accept-eula-for configuration option."""