diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..120eb9640a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, str): + if isinstance(paths, (str, Path)): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/test/framework/__init__.py b/test/framework/__init__.py index 895b15e622..04daadf7fa 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -27,3 +27,13 @@ @author: Toon Willems (Ghent University) """ + +from pathlib import Path +from easybuild.tools.filetools import read_file + +TEST_DIR = Path(__file__).parent +REPO_ROOT = TEST_DIR.parent.parent +TEST_MODULES_DIR = TEST_DIR / 'modules' +TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs' +TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb' +TOY_EC_TXT: str = read_file(TOY_EC) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 25760f1db8..3448ff6ffa 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -44,6 +44,7 @@ from inspect import cleandoc from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from unittest import TextTestRunner import easybuild.tools.systemtools as st @@ -172,8 +173,7 @@ def test_load_module(self): tmp_modules = os.path.join(self.test_prefix, 'modules') mkdir(tmp_modules) - test_dir = os.path.abspath(os.path.dirname(__file__)) - copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) + copy_dir(os.path.join(TEST_MODULES_DIR, 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '2.1.2-GCC-6.4.0-2.28') ompi_mod_txt = read_file(openmpi_module) @@ -223,12 +223,10 @@ def test_load_module(self): # test HMNS module load when conflicting dependencies are available in both Core and # toolchain-specific modulepaths # see also https://github.com/easybuilders/easybuild-framework/issues/4986 - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'easyconfigs', 'test_ecs') os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' build_options = { 'generate_devel_module': True, # go through EasyBlock.fake_module_environment() - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], } init_config(build_options=build_options) @@ -236,9 +234,8 @@ def test_load_module(self): mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'modules', 'HierarchicalMNS', mod_subdir) - copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir)) + copy_dir(os.path.join(TEST_MODULES_DIR, 'HierarchicalMNS', mod_subdir), + os.path.join(mod_prefix, mod_subdir)) # tweak use statements in toolchain module to ensure correct paths modfile = os.path.join(mod_prefix, 'Core', 'GCCcore', '12.3.0') @@ -248,7 +245,7 @@ def test_load_module(self): line) sys.stdout.write(line) - test_eb_file = os.path.join(test_ecs_path, 'g', 'GLib', 'GLib-2.77.1-GCCcore-12.3.0.eb') + test_eb_file = os.path.join(TEST_ECS_DIR, 'g', 'GLib', 'GLib-2.77.1-GCCcore-12.3.0.eb') eb = EasyBlock(EasyConfig(test_eb_file)) self.reset_modulepath([os.path.join(mod_prefix)]) @@ -1078,11 +1075,10 @@ def test_make_module_dep(self): def test_make_module_dep_hmns(self): """Test for make_module_dep under HMNS""" - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_stops': all_stops, 'validate': False, @@ -1123,11 +1119,10 @@ def test_make_module_dep_hmns(self): def test_make_module_dep_of_dep_hmns(self): """Test for make_module_dep under HMNS with dependencies of dependencies""" - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1203,11 +1198,8 @@ def test_det_iter_cnt(self): def test_handle_iterate_opts(self): """Test for handle_iterate_opts method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec) + "\nconfigopts = ['--opt1 --anotheropt', '--opt2', '--opt3 --optbis']") + write_file(test_ec, TOY_EC_TXT + "\nconfigopts = ['--opt1 --anotheropt', '--opt2', '--opt3 --optbis']") ec = process_easyconfig(test_ec)[0] eb = get_easyblock_instance(ec) @@ -1383,15 +1375,12 @@ def test_post_processing_step(self): """Test post_processing_step and deprecated post_install_step.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - # these imports only work here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy from easybuild.easyblocks.toy_deprecated import EB_toy_deprecated cwd = os.getcwd() - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy_deprecated(toy_ec) eb.silent = True depr_msg = r"EasyBlock.post_install_step\(\) is deprecated, use EasyBlock.post_processing_step\(\) instead" @@ -1400,7 +1389,7 @@ def test_post_processing_step(self): self.assertErrorRegex(EasyBuildError, expected_error, eb.run_all_steps, True) change_dir(cwd) - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy(toy_ec) eb.silent = True with self.mocked_stdout_stderr() as (_, stderr), self.saved_env(): @@ -1418,7 +1407,7 @@ def test_post_processing_step(self): change_dir(cwd) self.allow_deprecated_behaviour() - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy_deprecated(toy_ec) eb.silent = True with self.mocked_stdout_stderr() as (stdout, stderr), self.saved_env(): @@ -1527,8 +1516,7 @@ def test_extensions_step_deprecations(self): def test_init_extensions(self): """Test creating extension instances.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_txt = read_file(toy_ec_file) test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1992,9 +1980,8 @@ def test_make_builddir(self): def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" from easybuild.easyblocks.toy import EB_toy - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) self.assertIsInstance(eb, EB_toy) @@ -2006,11 +1993,10 @@ def test_get_easyblock_instance(self): def test_fetch_sources(self): """Test fetch_sources method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) - toy_source = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_source = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') with self.mocked_stdout_stderr(): eb.fetch_sources() @@ -2081,8 +2067,7 @@ def test_fetch_sources(self): def test_fetch_sources_git(self): """Test fetch_sources method from git repo.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) eb.src = [] sources = [ @@ -2223,8 +2208,7 @@ def test_download_instructions(self): def test_fetch_patches(self): """Test fetch_patches method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) toy_patch = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' @@ -2257,8 +2241,7 @@ def test_fetch_patches(self): self.assertEqual(eb.patches[3]['copy'], 'some/path') self.assertEqual(eb.patches[4]['name'], toy_patch) self.assertEqual(eb.patches[4]['level'], 0) - testdir = os.path.abspath(os.path.dirname(__file__)) - sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') + sandbox_sources = os.path.join(TEST_DIR, 'sandbox', 'sources') self.assertEqual(eb.patches[4]['path'], os.path.join(sandbox_sources, 'alt_toy', toy_patch)) patches = [ @@ -2269,8 +2252,7 @@ def test_fetch_patches(self): def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' - testdir = os.path.abspath(os.path.dirname(__file__)) - sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') + sandbox_sources = os.path.join(TEST_DIR, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) alt_toy_tarball_path = os.path.join(sandbox_sources, 'alt_toy', toy_tarball) tmpdir = tempfile.mkdtemp() @@ -2278,15 +2260,14 @@ def test_obtain_file(self): mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(tmpdir, 'ecs', 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) ec = process_easyconfig(test_ec)[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works - init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) + init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, TEST_DIR)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) with self.mocked_stdout_stderr(): res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) @@ -2428,10 +2409,9 @@ def test_fallback_source_url(self): def test_collect_exts_file_info(self): """Test collect_exts_file_info method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy') + toy_sources = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') toy_ext_sources = os.path.join(toy_sources, 'extensions') - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') new_ext_txt = "('baz', '0.0', {'nosource': True})," # With nosource option @@ -2526,8 +2506,7 @@ def test_collect_exts_file_info(self): def test_obtain_file_extension(self): """Test use of obtain_file method on an extension.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec = process_easyconfig(toy_ec_file)[0] toy_eb = EasyBlock(toy_ec['ec']) @@ -2548,8 +2527,7 @@ def test_check_readiness(self): # check that check_readiness step works (adding dependencies, etc.) ec_file = 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb' - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'o', 'OpenMPI', ec_file) + ec_path = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', ec_file) ec = EasyConfig(ec_path) eb = EasyBlock(ec) eb.check_readiness_step() @@ -2575,11 +2553,10 @@ def test_exclude_path_to_top_of_module_tree(self): w.r.t. not including any load statements for modules that build up the path to the top of the module tree. """ self.orig_module_naming_scheme = config.get_module_naming_scheme() - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -2603,7 +2580,7 @@ def test_exclude_path_to_top_of_module_tree(self): ('i/imkl/imkl-11.3.1.150-iimpi-2016.01.eb', imkl_modfile_path, iccifort_mods + ['iimpi', 'impi']), ] for ec_file, modfile_path, excluded_deps in tests: - ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, ec_file)) eb = EasyBlock(ec) with self.mocked_stdout_stderr(): eb.toolchain.prepare() @@ -2635,9 +2612,7 @@ def test_patch_step(self): """Test patch step.""" cwd = os.getcwd() - testdir = os.path.abspath(os.path.dirname(__file__)) - test_easyconfigs = os.path.join(testdir, 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]['ec'] + ec = process_easyconfig(TOY_EC)[0]['ec'] orig_sources = ec['sources'][:] toy_patches = [ @@ -2700,8 +2675,7 @@ def test_extensions_sanity_check(self): """Test sanity check aspect of extensions.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_fn = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # Do this before loading the easyblock to check the non-translated output below os.environ['LC_ALL'] = 'C' @@ -2739,29 +2713,25 @@ def test_extensions_sanity_check(self): def test_parallel(self): """Test defining of parallelism.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toytxt = read_file(toy_ec) - handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec1, toytxt + "\nparallel = 13") + write_file(toy_ec1, TOY_EC_TXT + "\nparallel = 13") handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 12\nmaxparallel = 6") + write_file(toy_ec2, TOY_EC_TXT + "\nparallel = 12\nmaxparallel = 6") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec3, toytxt + "\nparallel = False") + write_file(toy_ec3, TOY_EC_TXT + "\nparallel = False") handle, toy_ec4 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec4, toytxt + "\nmaxparallel = 6") + write_file(toy_ec4, TOY_EC_TXT + "\nmaxparallel = 6") handle, toy_ec5 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec5, toytxt + "\nmaxparallel = False") + write_file(toy_ec5, TOY_EC_TXT + "\nmaxparallel = False") # default: parallelism is derived from # available cores + ulimit # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here @@ -2787,7 +2757,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2823,7 +2793,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2860,7 +2830,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2878,7 +2848,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2888,7 +2858,7 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['parallel'], expected) # Template updated correctly - self.contents = toytxt + '\nmaxparallel=2' + self.contents = TOY_EC_TXT + '\nmaxparallel=2' self.writeEC() test_eb = EasyBlock(EasyConfig(self.eb_file)) test_eb.post_init() @@ -2906,7 +2876,7 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['buildopts'], '-j 1') # Legacy behavior. To be removed after deprecation of the parallel EC parameter - self.contents = toytxt + '\nmaxparallel=99' + self.contents = TOY_EC_TXT + '\nmaxparallel=99' self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2926,10 +2896,6 @@ def test_parallel(self): def test_keepsymlinks(self): """Test keepsymlinks parameter (default: True).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toytxt = read_file(toy_ec) - test_cases = { '': True, 'keepsymlinks = False': False, @@ -2938,7 +2904,7 @@ def test_keepsymlinks(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() test_eb = EasyBlock(EasyConfig(self.eb_file)) test_eb.post_init() @@ -2946,8 +2912,7 @@ def test_keepsymlinks(self): def test_guess_start_dir(self): """Test guessing the start dir.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] cwd = os.getcwd() self.assertExists(cwd) @@ -2984,8 +2949,7 @@ def check_start_dir(expected_start_dir): def test_extension_set_start_dir(self): """Test start dir with extensions.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] cwd = os.getcwd() self.assertExists(cwd) @@ -3090,8 +3054,7 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=Non def test_extension_patch_step(self): """Test start dir with extensions.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]['ec'] + ec = process_easyconfig(TOY_EC)[0]['ec'] cwd = os.getcwd() self.assertExists(cwd) @@ -3152,8 +3115,7 @@ def run_extension_step(): def test_prepare_step(self): """Test prepare step (setting up build environment).""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'system-system'), parents=True) eb = EasyBlock(ec['ec']) @@ -3181,8 +3143,7 @@ def test_prepare_step_load_tc_deps_modules(self): init_config(build_options={'robot_path': os.environ['EASYBUILD_ROBOT_PATHS']}) - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ompi_ec_file = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + ompi_ec_file = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') ec = process_easyconfig(ompi_ec_file, validate=False)[0] mkdir(os.path.join(self.test_buildpath, 'OpenMPI', '2.1.2', 'GCC-6.4.0-2.28'), parents=True) @@ -3215,10 +3176,9 @@ def test_prepare_step_hmns(self): """ Check whether loading of already existing dependencies during prepare step works when HierarchicalMNS is used. """ - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' - init_config(build_options={'robot_path': [test_ecs]}) + init_config(build_options={'robot_path': [TEST_ECS_DIR]}) # set up hierarchical modules, but reset $MODULEPATH to empty # the expectation is that EasyBuild set's up the $MODULEPATH such that pre-installed dependencies can be loaded @@ -3230,11 +3190,9 @@ def test_prepare_step_hmns(self): self.reset_modulepath([]) self.assertEqual(os.environ.get('MODULEPATH'), None) - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') regex = re.compile('^toolchain = .*', re.M) - test_ectxt = regex.sub("toolchain = SYSTEM", read_file(toy_ec)) + test_ectxt = regex.sub("toolchain = SYSTEM", TOY_EC_TXT) test_ectxt += "\ndependencies = [('GCC', '6.4.0', '-2.28')]" write_file(test_ec, test_ectxt) @@ -3254,9 +3212,7 @@ def test_prepare_step_cuda_cache(self): init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode - test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - ec = process_easyconfig(toy_ec)[0] + ec = process_easyconfig(TOY_EC)[0] eb = EasyBlock(ec['ec']) eb.silent = True with self.mocked_stdout_stderr(): @@ -3270,7 +3226,7 @@ def test_prepare_step_cuda_cache(self): # Now with CUDA test_ec = os.path.join(self.test_prefix, 'test.eb') test_ectxt = re.sub('^toolchain = .*', "toolchain = {'name': 'gcccuda', 'version': '2018a'}", - read_file(toy_ec), flags=re.M) + TOY_EC_TXT, flags=re.M) write_file(test_ec, test_ectxt) ec = process_easyconfig(test_ec)[0] eb = EasyBlock(ec['ec']) @@ -3310,8 +3266,7 @@ def test_prepare_step_cuda_cache(self): def test_checksum_step(self): """Test checksum step""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = process_easyconfig(toy_ec)[0] eb = get_easyblock_instance(ec) @@ -3362,7 +3317,7 @@ def test_checksum_step(self): # create test easyconfig from which checksums have been stripped test_ec = os.path.join(self.test_prefix, 'test.eb') - ectxt = read_file(toy_ec) + ectxt = TOY_EC_TXT regex = re.compile(r"'?checksums'?\s*[=:]\s*\[[^]]+\].*", re.M) ectxt = regex.sub('', ectxt) write_file(test_ec, ectxt) @@ -3380,7 +3335,7 @@ def test_checksum_step(self): self.fail("Incorrect extension type: %s" % type(ext)) # put checksums.json in place next to easyconfig file being used for the tests - toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums.json') + toy_checksums_json = os.path.join(TEST_ECS_DIR, 't', 'toy', 'checksums.json') copy_file(toy_checksums_json, os.path.join(self.test_prefix, 'checksums.json')) # test without checksums, it should work since they are in checksums.json @@ -3436,8 +3391,7 @@ def test_checksum_step(self): def test_check_checksums(self): """Test for check_checksums_for and check_checksums methods.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = process_easyconfig(toy_ec)[0] eb = get_easyblock_instance(ec) @@ -3465,9 +3419,7 @@ def run_checks(): self.assertIn(ext_error_tmpl % ext, line) # check whether tuple of alternative SHA256 checksums is correctly recognized - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - ec = process_easyconfig(toy_ec)[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) # single SHA256 checksum per source/patch: OK @@ -3545,7 +3497,7 @@ def run_checks(): # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) ec = process_easyconfig(test_ec)[0] eb = get_easyblock_instance(ec) eb.cfg['checksums'] = [] @@ -3556,7 +3508,7 @@ def run_checks(): self.assertEqual(res[0], expected) # all is fine if checksums.json is also copied - copy_file(os.path.join(os.path.dirname(toy_ec), 'checksums.json'), self.test_prefix) + copy_file(os.path.join(os.path.dirname(TOY_EC), 'checksums.json'), self.test_prefix) eb.json_checksums = None self.assertEqual(eb.check_checksums(), []) @@ -3763,9 +3715,7 @@ def test_arch_specific_sanity_check(self): def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) eb.dry_run = True error_pattern = r"Incorrect format for sanity_check_paths: " @@ -3870,9 +3820,6 @@ def test_report_current_step_method(self): """ Check whether name of methods in installation steps are correctly reported """ - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - class MockEasyBlock(EasyBlock): # Mock methods def build_step(self): @@ -3885,7 +3832,7 @@ def test_step(self): def custom_step(self): self.log.info('Ran custom') - eb = MockEasyBlock(EasyConfig(toy_ec)) + eb = MockEasyBlock(EasyConfig(TOY_EC)) # Part of run_all_steps steps = [step for step in eb.get_steps() if step[0] == BUILD_STEP] for step_name, _, step_methods, _ in steps: @@ -3911,10 +3858,8 @@ def test_exts_deps_build_env(self): Test whether dependencies are loaded in build environment for extensions. """ # to verify fix made in https://github.com/easybuilders/easybuild-framework/pull/5023 - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += textwrap.dedent(""" toolchain = {'name': 'gompi', 'version': '2023a'} @@ -3946,7 +3891,7 @@ def test_exts_deps_build_env(self): test_mods = os.path.join(self.test_prefix, 'modules') for name, mod_fn in mod_files: - mod_fp = os.path.join(testdir, 'modules', name, mod_fn) + mod_fp = os.path.join(TEST_MODULES_DIR, name, mod_fn) header_fn = 'zlib.h' if name == 'zlib' else 'mpi.h' @@ -4006,7 +3951,7 @@ def test_exts_deps_build_env(self): self.assertRegex(log_txt, regex) # verify fix made in https://github.com/easybuilders/easybuild-framework/pull/5048 - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += textwrap.dedent(""" toolchain = {'name': 'GCCcore', 'version': '12.3.0'} """) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96b7f47307..76fcf96f7d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -40,6 +40,7 @@ import textwrap from collections import OrderedDict from easybuild.tools import LooseVersion +from test.framework import TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -460,10 +461,9 @@ def test_extra_options(self): def test_exts_list(self): """Test handling of list of extensions.""" - topdir = os.path.dirname(os.path.abspath(__file__)) os.environ['EASYBUILD_SOURCEPATH'] = ':'.join([ - os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip'), - os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy'), + os.path.join(TEST_ECS_DIR, 'g', 'gzip'), + os.path.join(TEST_ECS_DIR, 't', 'toy'), ]) init_config() self.contents = textwrap.dedent(""" @@ -850,11 +850,9 @@ def test_alt_easyconfig_paths(self): def test_tweak_multiple_tcs(self): """Test that tweaking variables of ECs from multiple toolchains works""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # Create directories to store the tweaked easyconfigs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + robot_path = det_robot_path([TEST_ECS_DIR], tweaked_ecs_paths, pr_path, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), @@ -863,8 +861,8 @@ def test_tweak_multiple_tcs(self): }) # Allow tweaking of non-toolchain values for multiple ECs of different toolchains - untweaked_openmpi_1 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') - untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') + untweaked_openmpi_1 = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi_2 = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)]) tweak_specs = {'moduleclass': 'debugger'} easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths, @@ -1327,8 +1325,7 @@ def test_templating_constants(self): self.assertEqual(ec['source_urls'][3], 'https://github.com/pi/pi/releases/download/v3.04') # test use of %(mpi_cmd_prefix)s template - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - gompi_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a.eb') + gompi_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, read_file(gompi_ec) + "\nsanity_check_commands = ['%(mpi_cmd_prefix)s toy']") @@ -1509,8 +1506,7 @@ def test_templating_doc(self): def test_start_dir_template(self): """Test the %(startdir)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] self.contents = textwrap.dedent(""" name = 'toy' @@ -1554,11 +1550,9 @@ def test_start_dir_template(self): def test_rpath_template(self): """Test the %(rpath)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "configopts = '--with-rpath=%(rpath_enabled)s'" write_file(test_ec, test_ec_txt) @@ -1579,11 +1573,8 @@ def test_rpath_template(self): def test_sysroot_template(self): """Test the %(sysroot)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"' test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"' @@ -1607,11 +1598,8 @@ def test_sysroot_template(self): def test_software_commit_template(self): """Test the %(software_commit)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"' test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"' test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"' @@ -1794,7 +1782,7 @@ def test_format_equivalence_basic(self): orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True - easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs') + easyconfigs_path = TEST_DIR / 'easyconfigs' # set max diff high enough to make sure the difference is shown in case of problems self.maxDiff = 10000 @@ -1825,19 +1813,16 @@ def test_format_equivalence_basic(self): def test_fetch_parameters_from_easyconfig(self): """Test fetch_parameters_from_easyconfig function.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_file = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - for ec_file, correct_name, correct_easyblock in [ - (toy_ec_file, 'toy', None), - (os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb'), 'foss', 'Toolchain'), + (TOY_EC, 'toy', None), + (os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb'), 'foss', 'Toolchain'), ]: name, easyblock = fetch_parameters_from_easyconfig(read_file(ec_file), ['name', 'easyblock']) self.assertEqual(name, correct_name) self.assertEqual(easyblock, correct_easyblock) expected = "Toy C program, 100% toy." - self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], expected) + self.assertEqual(fetch_parameters_from_easyconfig(TOY_EC_TXT, ['description'])[0], expected) res = fetch_parameters_from_easyconfig("easyblock = 'ConfigureMake' # test comment", ['easyblock']) self.assertEqual(res, ['ConfigureMake']) @@ -1903,32 +1888,30 @@ def test_easyconfig_paths(self): def test_toolchain_inspection(self): """Test whether available toolchain inspection functionality is working.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) - ec = EasyConfig(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) tc_compilers = ['/'.join([x['name'], x['version']]) for x in det_toolchain_compilers(ec)] self.assertEqual(tc_compilers, ['GCC/6.4.0-2.28']) self.assertEqual(det_toolchain_mpi(ec)['name'], 'OpenMPI') - ec = EasyConfig(os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) tc_comps = det_toolchain_compilers(ec) expected = ['GCC/6.4.0-2.28'] self.assertEqual(['/'.join([x['name'], x['version'] + x['versionsuffix']]) for x in tc_comps], expected) self.assertEqual(det_toolchain_mpi(ec), None) - ec = EasyConfig(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) self.assertEqual(det_toolchain_compilers(ec), None) self.assertEqual(det_toolchain_mpi(ec), None) def test_filter_deps(self): """Test filtered dependencies.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') ec = EasyConfig(ec_file) self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) @@ -1953,13 +1936,13 @@ def test_filter_deps(self): build_options = { 'external_modules_metadata': ConfigObj(), 'minimal_toolchains': True, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) ec_file = os.path.join(self.test_prefix, 'test.eb') - shutil.copy2(os.path.join(test_ecs_dir, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb'), ec_file) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb'), ec_file) ec_txt = read_file(ec_file) ec_txt = ec_txt.replace('hwloc', 'deptobefiltered') @@ -1976,8 +1959,7 @@ def test_filter_deps(self): def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) replaced_parameters = { 'license': ('license_file', '2.0'), 'makeopts': ('buildopts', '2.0'), @@ -1996,10 +1978,7 @@ def foo(key): def test_alternative_easyconfig_parameters(self): """Test handling of alternative easyconfig parameters.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds') test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class') @@ -2045,8 +2024,7 @@ def test_deprecated_easyconfig_parameters(self): self.allow_deprecated_behaviour() init_config() - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = { 'foobar': ('barfoo', '0.0'), # deprecated since forever @@ -2097,9 +2075,8 @@ def foo(key): self.assertEqual(ec_params, expected) # try parsing an easyconfig file that defines a deprecated easyconfig parameter - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec)) + write_file(test_ec, TOY_EC_TXT) write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True) with self.mocked_stdout_stderr(): @@ -2130,8 +2107,7 @@ def set_ec_key(key): def test_external_dependencies(self): """Test specifying external (build) dependencies.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')) + ectxt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb')) toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') # just specify some of the test modules we ship, doesn't matter where they come from @@ -2385,10 +2361,6 @@ def test_external_dependencies(self): def test_external_dependencies_templates(self): """Test use of templates for dependencies marked as external modules.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ectxt = read_file(toy_ec) - extra_ectxt = '\n'.join([ "versionsuffix = '-Python-%(pyver)s-Perl-%(perlshortver)s'", '', @@ -2398,7 +2370,7 @@ def test_external_dependencies_templates(self): "]", ]) test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, toy_ectxt + '\n' + extra_ectxt) + write_file(test_ec, TOY_EC_TXT + '\n' + extra_ectxt) # put metadata in place so templates can be defined metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') @@ -2437,8 +2409,7 @@ def test_external_dependencies_templates(self): def test_update(self): """Test use of update() method for EasyConfig instances.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ebfile = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ebfile = TOY_EC ec = EasyConfig(toy_ebfile) # for string values: append @@ -2481,8 +2452,7 @@ def test_update(self): def test_hide_hidden_deps(self): """Test use of --hide-deps on hiddendependencies.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = TEST_ECS_DIR / 'g' / 'gzip' / 'gzip-1.4-GCC-4.6.3.eb' ec = EasyConfig(ec_file) self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') self.assertEqual(ec['dependencies'][0]['full_mod_name'], 'toy/.0.0-deps') @@ -2588,10 +2558,9 @@ def subtest_quote_py_str(val): def test_dump(self): """Test EasyConfig's dump() method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -2605,7 +2574,7 @@ def test_dump(self): for ecfile in ecfiles: test_ec = os.path.join(self.test_prefix, 'test.eb') - ec = EasyConfig(os.path.join(test_ecs_dir, ecfile)) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, ecfile)) with ec.disable_templating(): ecdict = ec.asdict() ec.dump(test_ec) @@ -2676,10 +2645,9 @@ def test_dump(self): def test_toolchain_hierarchy_aware_dump(self): """Test that EasyConfig's dump() method is aware of the toolchain hierarchy.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -3283,16 +3251,15 @@ def test_dep_graph(self): print("Skipping test_dep_graph, since graphviz is not available") return - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, } init_config(build_options=build_options) - ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') ec_files = [(ec_file, False)] ecs, _ = parse_easyconfigs(ec_files) @@ -3326,20 +3293,16 @@ def test_dep_graph_multi_deps(self): print("Skipping test_dep_graph_multi_deps, since graphviz is not available") return - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, } init_config(build_options=build_options) - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = toy_ec_txt + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" + test_ec_txt = TOY_EC_TXT + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" write_file(test_ec, test_ec_txt) ec_files = [(test_ec, False)] @@ -3379,15 +3342,14 @@ def test_ActiveMNS_det_full_module_name(self): } init_config(build_options=build_options) - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + ec_file = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-deps.eb' ec = EasyConfig(ec_file) self.assertEqual(ActiveMNS().det_full_module_name(ec), 'toy/0.0-deps') self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][0]), 'intel/2018a') self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][1]), 'GCC/6.4.0-2.28') - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ec = EasyConfig(ec_file) hiddendep = ec['hiddendependencies'][0] self.assertEqual(ActiveMNS().det_full_module_name(hiddendep), 'toy/.0.0-deps') @@ -3395,46 +3357,45 @@ def test_ActiveMNS_det_full_module_name(self): def test_find_related_easyconfigs(self): """Test find_related_easyconfigs function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') ec = EasyConfig(ec_file) # exact match: GCC-4.6.3.eb - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['GCC-4.6.3.eb']) # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match ec['version'] = '4.6.1' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['GCC-4.6.4.eb', 'GCC-4.6.3.eb']) # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match ec['version'] = '4.5.0' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] expected = ['GCC-4.9.2.eb', 'GCC-4.8.3.eb', 'GCC-4.8.2.eb', 'GCC-4.6.4.eb', 'GCC-4.6.3.eb'] self.assertEqual(res, expected) - ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') ec = EasyConfig(ec_file) # exact match - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-deps.eb']) # tweak toolchain name/version and versionsuffix => closest match with same toolchain name is found ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} ec['versionsuffix'] = '-foobar' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb']) # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) ec['versionsuffix'] = '-deps' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-deps.eb']) # no matches for unknown software name ec['name'] = 'nosuchsoftware' - self.assertEqual(find_related_easyconfigs(test_easyconfigs, ec), []) + self.assertEqual(find_related_easyconfigs(TEST_ECS_DIR, ec), []) # no problem with special characters in software name ec['name'] = 'nosuchsoftware++' @@ -3445,8 +3406,7 @@ def test_find_related_easyconfigs(self): def test_modaltsoftname(self): """Test specifying an alternative name for the software name, to use when determining module name.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + ec_file = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-deps.eb' ectxt = read_file(ec_file) modified_ec_file = os.path.join(self.test_prefix, os.path.basename(ec_file)) write_file(modified_ec_file, ectxt + "\nmodaltsoftname = 'notreallyatoy'") @@ -3458,15 +3418,14 @@ def test_modaltsoftname(self): def test_software_license(self): """Tests related to software_license easyconfig parameter.""" # default: None - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC ec = EasyConfig(ec_file) ec.validate_license() self.assertEqual(ec['software_license'], None) self.assertEqual(ec.software_license, None) # specified software license gets handled correctly - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') ec = EasyConfig(ec_file) ec.validate_license() # constant GPLv3 is resolved as string @@ -3481,8 +3440,7 @@ def test_software_license(self): def test_param_value_type_checking(self): """Test value tupe checking of easyconfig parameters.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb') + ec_file = TEST_ECS_DIR / 'g' / 'gzip' / 'gzip-1.4-broken.eb' # version parameter has values of wrong type in this broken easyconfig error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*" self.assertErrorRegex(EasyBuildError, error_msg_pattern, EasyConfig, ec_file, auto_convert_value_types=False) @@ -3495,8 +3453,7 @@ def test_copy(self): """Test copy method of EasyConfig object.""" init_config(build_options={'silent': True}) - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec1 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) + ec1 = EasyConfig(TOY_EC) # inject fake template value, just to check whether they are copied over too ec1.template_values['pyshortver'] = '3.7' @@ -3511,9 +3468,8 @@ def test_copy(self): def test_eq_hash(self): """Test comparing two EasyConfig instances.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec1 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) - ec2 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) + ec1 = EasyConfig(TOY_EC) + ec2 = EasyConfig(TOY_EC) # different instances, same parsed easyconfig self.assertIsNot(ec1, ec2) @@ -3525,14 +3481,13 @@ def test_eq_hash(self): self.assertEqual(hash(ec1), hash(ec2)) # other parsed easyconfig is not equal - ec3 = EasyConfig(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb')) + ec3 = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb')) self.assertFalse(ec1 == ec3) self.assertTrue(ec1 != ec3) def test_copy_easyconfigs(self): """Test copy_easyconfigs function.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') target_dir = os.path.join(self.test_prefix, 'copied_ecs') # easybuild/easyconfigs subdir is expected to exist @@ -3555,7 +3510,7 @@ def test_copy_easyconfigs(self): ecs_to_copy = [] for (src_ec, target_ec) in test_ecs: ecs_to_copy.append(os.path.join(self.test_prefix, target_ec)) - shutil.copy2(os.path.join(test_ecs_dir, src_ec), ecs_to_copy[-1]) + shutil.copy2(os.path.join(TEST_ECS_DIR, src_ec), ecs_to_copy[-1]) res = copy_easyconfigs(ecs_to_copy, target_dir) self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'new_file_in_existing_folder', @@ -3575,7 +3530,7 @@ def test_copy_easyconfigs(self): # create test easyconfig that includes comments & build stats, just like an archived easyconfig toy_ec = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'), toy_ec) + copy_file(TOY_EC, toy_ec) toy_ec_txt = read_file(toy_ec) toy_ec_txt = '\n'.join([ "# Built with EasyBuild version 3.1.2 on 2017-04-25_21-35-15", @@ -3618,8 +3573,7 @@ def test_copy_easyconfigs(self): def test_template_constant_dict(self): """Test template_constant_dict function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) arch_regex = re.compile('^[a-z0-9_]+$') @@ -3673,7 +3627,7 @@ def test_template_constant_dict(self): res.pop('arch') self.assertEqual(res, expected) - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') toy_ec_txt = read_file(toy_ec) # fiddle with version to check version_minor template ('0' should be retained) @@ -3803,10 +3757,8 @@ def test_template_constant_dict(self): def test_parse_deps_templates(self): """Test whether handling of templates defined by dependencies is done correctly.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - pyec = os.path.join(self.test_prefix, 'Python-2.7.10-foss-2018a.eb') - shutil.copy2(os.path.join(test_ecs, 'p', 'Python', 'Python-2.7.10-intel-2018a.eb'), pyec) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'p', 'Python', 'Python-2.7.10-intel-2018a.eb'), pyec) write_file(pyec, "\ntoolchain = {'name': 'foss', 'version': '2018a'}", append=True) ec_txt = '\n'.join([ @@ -3836,7 +3788,7 @@ def test_parse_deps_templates(self): build_options = { 'external_modules_metadata': ConfigObj(), - 'robot_path': [test_ecs, self.test_prefix], + 'robot_path': [TEST_ECS_DIR, self.test_prefix], 'valid_module_classes': module_classes(), 'validate': False, } @@ -3856,8 +3808,7 @@ def test_parse_deps_templates(self): def test_hidden_toolchain(self): """Test hiding of toolchain via easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_txt = read_file(os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')) + ec_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')) new_tc = "toolchain = {'name': 'GCC', 'version': '4.9.2', 'hidden': True}" ec_txt = re.sub("toolchain = .*", new_tc, ec_txt, re.M) @@ -3879,16 +3830,14 @@ def test_categorize_files_by_type(self): self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, categorize_files_by_type([])) - test_dir = os.path.dirname(os.path.abspath(__file__)) - test_ecs_dir = os.path.join(test_dir, 'easyconfigs') toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) - easyblocks_dir = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks') + easyblocks_dir = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') configuremake = os.path.join(easyblocks_dir, 'generic', 'configuremake.py') toy_easyblock = os.path.join(easyblocks_dir, 't', 'toy.py') - gzip_ec = os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') paths = [ 'bzip2-1.0.6.eb', toy_easyblock, @@ -4040,8 +3989,7 @@ def test_det_subtoolchain_version(self): def test_verify_easyconfig_filename(self): """Test verify_easyconfig_filename function""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_name = os.path.basename(toy_ec) specs = { 'name': 'toy', @@ -4094,16 +4042,14 @@ def test_get_paths_for(self): path.append(subdir) os.environ['PATH'] = os.pathsep.join(path) - top_dir = os.path.dirname(os.path.abspath(__file__)) mkdir(os.path.join(self.test_prefix, 'easybuild')) - test_ecs = os.path.join(top_dir, 'easyconfigs') - symlink(test_ecs, os.path.join(self.test_prefix, 'easybuild', 'easyconfigs')) + symlink(TEST_ECS_DIR, os.path.join(self.test_prefix, 'easybuild', 'easyconfigs')) # temporarily mock stderr to avoid printed warning (because 'eb' is not available via $PATH) with self.mocked_stderr(): # locations listed in 'robot_path' named argument are taken into account res = get_paths_for(subdir='easyconfigs', robot_path=[self.test_prefix]) - self.assertTrue(os.path.samefile(test_ecs, res[0])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[0])) # Can't have EB_SCRIPT_PATH set (for some of) these tests env_eb_script_path = os.getenv('EB_SCRIPT_PATH') @@ -4116,7 +4062,7 @@ def test_get_paths_for(self): os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), orig_path) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[-1])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[-1])) # also works when 'eb' resides in a symlinked location altbin = os.path.join(self.test_prefix, 'some', 'other', 'symlinked', 'bin') @@ -4124,7 +4070,7 @@ def test_get_paths_for(self): symlink(os.path.join(self.test_prefix, 'bin'), altbin) os.environ['PATH'] = '%s:%s' % (altbin, orig_path) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[-1])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[-1])) # Restore (temporarily) EB_SCRIPT_PATH value if set originally if env_eb_script_path: @@ -4134,7 +4080,7 @@ def test_get_paths_for(self): os.environ['PATH'] = orig_path sys.path.insert(0, self.test_prefix) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[0])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[0])) # put mock 'eb' back in $PATH os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), orig_path) @@ -4194,10 +4140,9 @@ def test_get_module_path(self): def test_not_an_easyconfig(self): """Test error reporting when a file that's not actually an easyconfig file is provided.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) # run test on an easyconfig file that was downloaded using wget using a non-raw GitHub URL # cfr. https://github.com/easybuilders/easybuild-framework/issues/2383 - not_an_ec = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'not_an_easyconfig.eb') + not_an_ec = TEST_DIR / 'sandbox' / 'not_an_easyconfig.eb' # from Python 3.10 onwards: invalid decimal literal # older Python versions: invalid syntax @@ -4206,9 +4151,7 @@ def test_not_an_easyconfig(self): def test_check_sha256_checksums(self): """Test for check_sha256_checksums function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT checksums_regex = re.compile(r'^checksums = \[\[(.|\n)*\]\]', re.M) @@ -4256,7 +4199,7 @@ def test_check_sha256_checksums(self): self.assertEqual(check_sha256_checksums(ecs), []) # also test toy easyconfig with extensions, for which some checksums are missing - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ecs, _ = parse_easyconfigs([(toy_ec, False)]) ecs = [ec['ec'] for ec in ecs] @@ -4295,8 +4238,7 @@ def test_check_sha256_checksums(self): def test_deprecated(self): """Test use of 'deprecated' easyconfig parameter.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_txt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')) + toy_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, toy_ec_txt + "\ndeprecated = 'this is just a test'") @@ -4311,8 +4253,7 @@ def test_deprecated(self): def test_deprecated_toolchain(self): """Test use of deprecated toolchain""" - topdir = os.path.dirname(os.path.abspath(__file__)) - deprecated_toolchain_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb') + deprecated_toolchain_ec = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-gompi-2018a.eb' init_config(build_options={'silence_deprecation_warnings': [], 'unit_testing_mode': False}) error_pattern = r"toolchain 'gompi/2018a' is marked as deprecated \(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, deprecated_toolchain_ec) @@ -4326,7 +4267,6 @@ def test_deprecated_toolchain(self): def test_filename(self): """Test filename method of EasyConfig class.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') test_ecs = [ os.path.join('g', 'GCC', 'GCC-4.6.4.eb'), @@ -4337,14 +4277,13 @@ def test_filename(self): os.path.join('t', 'toy', 'toy-0.0-deps.eb'), ] for test_ec in test_ecs: - test_ec = os.path.join(test_ecs_dir, test_ec) + test_ec = os.path.join(TEST_ECS_DIR, test_ec) ec = EasyConfig(test_ec) self.assertTrue(ec.filename(), os.path.basename(test_ec)) def test_get_ref(self): """Test get_ref method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-iter.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-iter.eb')) # without using get_ref, we get a (templated) copy rather than the original value sources = ec['sources'] @@ -4375,9 +4314,8 @@ def test_get_ref(self): def test_multi_deps(self): """Test handling of multi_deps easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec = TOY_EC + toy_ec_txt = TOY_EC_TXT ec = EasyConfig(toy_ec) self.assertEqual(ec['builddependencies'], []) @@ -4450,9 +4388,7 @@ def test_multi_deps(self): def test_multi_deps_templated_builddeps(self): """Test effect of multi_deps on builddependencies w.r.t. resolving templates like %(pyver)s.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = toy_ec_txt + "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}" @@ -4499,12 +4435,8 @@ def test_multi_deps_templated_builddeps(self): def test_iter_builddeps_templates(self): """Test whether iterative builddependencies are taken into account to define *ver and *shortver templates.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = toy_ec_txt + "\nmulti_deps = {'Python': ['2.7.15', '3.6.6']}" + test_ec_txt = TOY_EC_TXT + "\nmulti_deps = {'Python': ['2.7.15', '3.6.6']}" # inject extension that uses %(pyshortver)s, to check whether the template value is properly resolved test_ec_txt += '\n'.join([ @@ -4560,21 +4492,16 @@ def test_iter_builddeps_templates(self): def test_fix_deprecated_easyconfigs(self): """Test fix_deprecated_easyconfigs function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') # need to allow triggering deprecated behaviour, since that's exactly what we're fixing... self.allow_deprecated_behaviour() - test_ectxt = toy_ec_txt # inject local variables with names that need to be tweaked (or not for single-letter ones) regex = re.compile('^(sanity_check_paths)', re.M) # purposely define configopts via local variable 'foo', which has value that also contains 'foo' substring; # that way, we can check whether only the 'foo' variable name is replaced with 'local_foo' - test_ectxt = regex.sub(r'foo = "--foobar --barfoo --barfoobaz"\nconfigopts = foo\n\n\1', toy_ec_txt) + test_ectxt = regex.sub(r'foo = "--foobar --barfoo --barfoobaz"\nconfigopts = foo\n\n\1', TOY_EC_TXT) regex = re.compile(r'^(toolchain\s*=.*)$', re.M) test_ectxt = regex.sub(r'\1\n\nsome_list = [x + "1" for x in ["one", "two", "three"]]', test_ectxt) @@ -4969,8 +4896,7 @@ def test_det_copy_ec_specs(self): def test_recursive_module_unload(self): """Test use of recursive_module_unload easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) @@ -5086,8 +5012,7 @@ def test_pure_ec(self): Test whether we can get a 'pure' view on the easyconfig file, which correctly reflects what's defined in the easyconfig file. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + toy_ec = EasyConfig(TOY_EC) ec_dict = toy_ec.parser.get_config_dict() self.assertEqual(ec_dict.get('version'), '0.0') @@ -5111,11 +5036,9 @@ def test_easyconfig_import(self): """ Test parsing of an easyconfig file that includes import statements. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "import os", "local_test = os.getenv('TEST_TOY')", @@ -5250,11 +5173,8 @@ def test_get_amdgcn_cc_template_value(self): def test_count_files(self): """Tests for EasyConfig.count_files method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - foss = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') - toy = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_exts = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + foss = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + toy_exts = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # no sources or patches for toolchain => 0 foss_ec = EasyConfig(foss) @@ -5262,7 +5182,7 @@ def test_count_files(self): self.assertEqual(foss_ec['patches'], []) self.assertEqual(foss_ec.count_files(), 0) # 1 source + 2 patches => 3 - toy_ec = EasyConfig(toy) + toy_ec = EasyConfig(TOY_EC) self.assertEqual(len(toy_ec['sources']), 1) self.assertEqual(len(toy_ec['patches']), 2) self.assertEqual(toy_ec['exts_list'], []) @@ -5310,13 +5230,11 @@ def test_easyconfigs_caches(self): """ Test whether easyconfigs caches work as intended. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - libtoy_ec = os.path.join(test_ecs_dir, 'l', 'libtoy', 'libtoy-0.0.eb') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + libtoy_ec = os.path.join(TEST_ECS_DIR, 'l', 'libtoy', 'libtoy-0.0.eb') copy_file(libtoy_ec, self.test_prefix) - copy_file(toy_ec, self.test_prefix) + copy_file(TOY_EC, self.test_prefix) libtoy_ec = os.path.join(self.test_prefix, os.path.basename(libtoy_ec)) - toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) + toy_ec = os.path.join(self.test_prefix, os.path.basename(TOY_EC)) ec1 = process_easyconfig(toy_ec)[0] self.assertEqual(ec1['ec'].name, 'toy') @@ -5378,10 +5296,8 @@ def test_templates(self): """ Test use of template values like %(version)s """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\ndescription = "name: %(name)s, version: %(version)s"' test_ec = os.path.join(self.test_prefix, 'test.eb') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 18130d166a..88c83c8c27 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -57,6 +57,8 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import LINUX, get_os_type +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT + class FileToolsTest(EnhancedTestCase): """ Testcase for filetools module """ @@ -561,8 +563,7 @@ def test_download_file(self): fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_source_dir = os.path.join(test_dir, 'sandbox', 'sources', 'toy') + toy_source_dir = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') source_url = 'file://%s/%s' % (toy_source_dir, fn) with self.mocked_stdout_stderr(): res = ft.download_file(fn, source_url, target_location) @@ -587,11 +588,11 @@ def test_download_file(self): # non-existing files result in None return value with self.mocked_stdout_stderr(): - self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) + self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % TEST_DIR, target_location), None) # install broken proxy handler for opening local files # this should make urlopen use this broken proxy for downloading from a file:// URL - proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) + proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % TEST_DIR}) request.install_opener(request.build_opener(proxy_handler)) # for Python 3.14+, we need to make sure that proxy and original URL are using different protocol, @@ -727,8 +728,7 @@ def fake_urllib_open(url, *args, **kwargs): return self.orig_filetools_std_urllib_urlopen(url, *args, **kwargs) fn = 'toy-0.0.eb' - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_dir = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy') + toy_dir = os.path.join(TEST_ECS_DIR, 't', 'toy') url = 'file://%s/%s' % (toy_dir, fn) ft.std_urllib.urlopen = fake_urllib_open @@ -793,8 +793,7 @@ def test_download_file_fallback_source_urls(self): """ fn = 'toy-0.0.eb' - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_dir = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy') + toy_dir = os.path.join(TEST_ECS_DIR, 't', 'toy') correct_url = f'file://{toy_dir}/' wrong_url = f'file://{self.test_prefix}/easyconfigs/' @@ -1102,7 +1101,7 @@ def test_is_binary(self): def test_det_patched_files(self): """Test det_patched_files function.""" toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - pf = os.path.join(os.path.dirname(__file__), 'sandbox', 'sources', 'toy', toy_patch_fn) + pf = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) self.assertEqual(ft.det_patched_files(pf), ['b/toy-0.0/toy.source']) self.assertEqual(ft.det_patched_files(pf, omit_ab_prefix=True), ['toy-0.0/toy.source']) @@ -1304,15 +1303,13 @@ def test_move_logs(self): def test_multidiff(self): """Test multidiff function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') other_toy_ecs = [ - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb'), - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), ] # default (colored) - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - lines = multidiff(toy_ec, other_toy_ecs).split('\n') + lines = multidiff(TOY_EC, other_toy_ecs).split('\n') expected = "Comparing \x1b[0;35mtoy-0.0.eb\x1b[0m with toy-0.0-deps.eb, toy-0.0-gompi-2018a-test.eb" red = "\x1b[0;41m" @@ -1342,7 +1339,7 @@ def test_multidiff(self): self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) self.assertEqual(lines[-1], "=====") - lines = multidiff(toy_ec, other_toy_ecs, colored=False).split('\n') + lines = multidiff(TOY_EC, other_toy_ecs, colored=False).split('\n') self.assertEqual(lines[0], "Comparing toy-0.0.eb with toy-0.0-deps.eb, toy-0.0-gompi-2018a-test.eb") self.assertEqual(lines[1], "=====") @@ -1801,10 +1798,9 @@ def test_find_flexlm_license(self): def test_is_patch_file(self): """Test for is_patch_file() function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - self.assertFalse(ft.is_patch_file(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))) + self.assertFalse(ft.is_patch_file(TOY_EC)) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - self.assertTrue(ft.is_patch_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn))) + self.assertTrue(ft.is_patch_file(os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn))) def test_is_alt_pypi_url(self): """Test is_alt_pypi_url() function.""" @@ -1905,12 +1901,11 @@ def test_create_patch_info(self): def test_apply_patch(self): """ Test apply_patch """ - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_tar_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_tar_gz = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) for with_backup in (True, False): update_build_option('backup_patched_files', with_backup) @@ -1929,7 +1924,7 @@ def test_apply_patch(self): self.assertNotExists(backup_file) # This patch is dependent on the previous one - toy_patch_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz') + toy_patch_gz = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz') with self.mocked_stdout_stderr(): self.assertTrue(ft.apply_patch(toy_patch_gz, path)) patched_gz = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source')) @@ -2003,12 +1998,10 @@ def test_apply_patch(self): def test_copy_file(self): """Test copy_file function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(self.test_prefix, 'toy.eb') - ft.copy_file(toy_ec, target_path) + ft.copy_file(TOY_EC, target_path) self.assertExists(target_path) - self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) + self.assertEqual(ft.read_file(target_path), TOY_EC_TXT) # Make sure it doesn't fail if path is a symlink and target_path is a dir toy_link_fn = 'toy-link-0.0.eb' @@ -2032,7 +2025,7 @@ def test_copy_file(self): ft.remove_file(copied_file) # clean error when trying to copy a directory with copy_file - src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy') + src, target = os.path.dirname(TOY_EC), os.path.join(self.test_prefix, 'toy') # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist" error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target) @@ -2082,7 +2075,7 @@ def test_copy_file(self): self.assertNotExists(target_path) with self.mocked_stdout(): - ft.copy_file(toy_ec, target_path) + ft.copy_file(TOY_EC, target_path) txt = self.get_stdout() self.assertNotExists(target_path) @@ -2090,11 +2083,11 @@ def test_copy_file(self): # forced copy, even in dry run mode with self.mocked_stdout(): - ft.copy_file(toy_ec, target_path, force_in_dry_run=True) + ft.copy_file(TOY_EC, target_path, force_in_dry_run=True) txt = self.get_stdout() self.assertExists(target_path) - self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) + self.assertEqual(ft.read_file(target_path), TOY_EC_TXT) self.assertEqual(txt, '') # Test that a non-existing file raises an exception @@ -2155,18 +2148,15 @@ def test_copy_file_xattr(self): def test_copy_files(self): """Test copy_files function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = ft.read_file(toy_ec) - bzip2_ec = os.path.join(test_ecs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') + bzip2_ec = os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') bzip2_ec_txt = ft.read_file(bzip2_ec) # copying a single file to a non-existing directory target_dir = os.path.join(self.test_prefix, 'target_dir1') - ft.copy_files([toy_ec], target_dir) + ft.copy_files([TOY_EC], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) # copying a single file to an existing directory ft.copy_files([bzip2_ec], target_dir) @@ -2176,10 +2166,10 @@ def test_copy_files(self): # copying multiple files to a non-existing directory target_dir = os.path.join(self.test_prefix, 'target_dir_multiple') - ft.copy_files([toy_ec, bzip2_ec], target_dir) + ft.copy_files([TOY_EC, bzip2_ec], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) copied_bzip2_ec = os.path.join(target_dir, 'bzip2-1.0.6-GCC-4.9.2.eb') self.assertExists(copied_bzip2_ec) self.assertEqual(ft.read_file(copied_bzip2_ec), bzip2_ec_txt) @@ -2197,10 +2187,10 @@ def test_copy_files(self): # test special case: copying a single file to a file target via target_single_file=True target = os.path.join(self.test_prefix, 'target') self.assertNotExists(target) - ft.copy_files([toy_ec], target, target_single_file=True) + ft.copy_files([TOY_EC], target, target_single_file=True) self.assertExists(target) self.assertTrue(os.path.isfile(target)) - self.assertEqual(toy_ec_txt, ft.read_file(target)) + self.assertEqual(ft.read_file(target), TOY_EC_TXT) ft.remove_file(target) @@ -2208,27 +2198,27 @@ def test_copy_files(self): target = os.path.join(self.test_prefix, 'target_parent', 'target_subdir', 'target.txt') self.assertNotExists(target) self.assertNotExists(os.path.dirname(target)) - ft.copy_files([toy_ec], target, target_single_file=True) + ft.copy_files([TOY_EC], target, target_single_file=True) self.assertExists(target) self.assertTrue(os.path.isfile(target)) - self.assertEqual(toy_ec_txt, ft.read_file(target)) + self.assertEqual(ft.read_file(target), TOY_EC_TXT) ft.remove_file(target) # default behaviour is to copy single file list to target *directory* self.assertNotExists(target) - ft.copy_files([toy_ec], target) + ft.copy_files([TOY_EC], target) self.assertExists(target) self.assertTrue(os.path.isdir(target)) copied_toy_ec = os.path.join(target, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) ft.remove_dir(target) # test enabling verbose mode with self.mocked_stdout_stderr(): - ft.copy_files([toy_ec], target, verbose=True) + ft.copy_files([TOY_EC], target, verbose=True) stderr, stdout = self.get_stderr(), self.get_stdout() self.assertEqual(stderr, '') regex = re.compile(r"^1 file\(s\) copied to .*/target") @@ -2237,7 +2227,7 @@ def test_copy_files(self): ft.remove_dir(target) with self.mocked_stdout_stderr(): - ft.copy_files([toy_ec], target, target_single_file=True, verbose=True) + ft.copy_files([TOY_EC], target, target_single_file=True, verbose=True) stderr, stdout = self.get_stderr(), self.get_stdout() self.assertEqual(stderr, '') regex = re.compile(r"/.*/toy-0\.0\.eb copied to .*/target") @@ -2312,8 +2302,7 @@ def test_has_recursive_symlinks(self): def test_copy_dir(self): """Test copy_dir function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') + to_copy = os.path.join(TEST_ECS_DIR, 'g', 'GCC') target_dir = os.path.join(self.test_prefix, 'GCC') self.assertNotExists(target_dir) @@ -2419,12 +2408,10 @@ def ignore_func(_, names): def test_copy(self): """Test copy function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - - toy_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_file = TOY_EC toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) - gcc_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) + gcc_dir = os.path.join(TEST_ECS_DIR, 'g', 'GCC') ft.copy([toy_file, gcc_dir, toy_patch], self.test_prefix) @@ -2506,8 +2493,7 @@ def test_extract_file(self): """Test extract_file""" cwd = os.getcwd() - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_tarball = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_tarball = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) with self.mocked_stdout_stderr(): @@ -2708,23 +2694,20 @@ def test_clean_dir(self): def test_index_functions(self): """Test *_index functions.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # create_index checks whether specified path is an existing directory doesnotexist = os.path.join(self.test_prefix, 'doesnotexist') self.assertErrorRegex(EasyBuildError, "Specified path does not exist", ft.create_index, doesnotexist) - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, toy_ec) + self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, TOY_EC) # load_index just returns None if there is no index in specified directory self.assertEqual(ft.load_index(self.test_prefix), None) - num_files = len(glob.glob(test_ecs + '/**/*.*', recursive=True)) + num_files = len(list(TEST_ECS_DIR.rglob('*.*'))) # create index for test easyconfigs; # test with specified path with and without trailing '/'s - for path in [test_ecs, test_ecs + '/', test_ecs + '//']: + for path in [TEST_ECS_DIR, str(TEST_ECS_DIR), str(TEST_ECS_DIR) + '/', str(TEST_ECS_DIR) + '//']: index = ft.create_index(path) self.assertEqual(len(index), num_files) @@ -2741,7 +2724,7 @@ def test_index_functions(self): # set up some files to create actual index file for ecs_dir = os.path.join(self.test_prefix, 'easyconfigs') - ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir) + ft.copy_dir(os.path.join(TEST_ECS_DIR, 'g'), ecs_dir) # test dump_index function index_fp = ft.dump_index(ecs_dir) @@ -2824,10 +2807,9 @@ def test_index_functions(self): def test_search_file(self): """Test search_file function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # check for default semantics, test case-insensitivity - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True) self.assertEqual(var_defs, []) self.assertEqual(len(hits), 5) self.assertTrue(all(os.path.exists(p) for p in hits)) @@ -2838,16 +2820,16 @@ def test_search_file(self): self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching - var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) self.assertEqual(hits_case_sensitive, []) - var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([TEST_ECS_DIR], 'hwloc', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) self.assertEqual(hits_case_sensitive, hits) # check filename-only mode - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', 'hwloc-1.8-gcccuda-2018a.eb', @@ -2857,12 +2839,12 @@ def test_search_file(self): ]) # check specifying of ignored dirs - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, ignore_dirs=['hwloc']) self.assertEqual(var_defs + hits, []) # check short mode - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) - self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, short=True) + self.assertEqual(var_defs, [('CFGS1', os.path.join(TEST_ECS_DIR, 'h', 'hwloc'))]) self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', '$CFGS1/hwloc-1.8-gcccuda-2018a.eb', '$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', @@ -2871,19 +2853,19 @@ def test_search_file(self): ]) # check terse mode (implies 'silent', overrides 'short') - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', terse=True, short=True) self.assertEqual(var_defs, []) expected = [ - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', 'hwloc-1.8-gcccuda-2018a.eb', @@ -2895,7 +2877,7 @@ def test_search_file(self): # patterns that include special characters + (or ++) shouldn't cause trouble # cfr. https://github.com/easybuilders/easybuild-framework/issues/2966 for pattern in ['netCDF-C++', 'foo.*bar', 'foo|bar']: - var_defs, hits = ft.search_file([test_ecs], pattern, terse=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], pattern, terse=True, filename_only=True) self.assertEqual(var_defs, []) # no hits for any of these in test easyconfigs self.assertEqual(hits, []) @@ -2904,7 +2886,7 @@ def test_search_file(self): # to avoid accidental matches in other files already present (log files, etc.) ec_dir = tempfile.mkdtemp() test_ec = os.path.join(ec_dir, 'netCDF-C++-4.2-foss-2019a.eb') - ft.write_file(test_ec, ''), + ft.write_file(test_ec, '') for pattern in ['netCDF-C++', 'CDF', 'C++', '^netCDF']: var_defs, hits = ft.search_file([ec_dir], pattern, terse=True, filename_only=True) self.assertEqual(var_defs, [], msg='For pattern ' + pattern) @@ -2912,7 +2894,7 @@ def test_search_file(self): # check how simply invalid queries are handled for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']: - self.assertErrorRegex(EasyBuildError, "Invalid search query", ft.search_file, [test_ecs], pattern) + self.assertErrorRegex(EasyBuildError, "Invalid search query", ft.search_file, [TEST_ECS_DIR], pattern) def test_dir_contains_files(self): def makedirs_in_test(*paths): @@ -3564,8 +3546,7 @@ def foobar(): def test_get_easyblock_class_name(self): """Test for get_easyblock_class_name function.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') self.assertEqual(ft.get_easyblock_class_name(configuremake), 'ConfigureMake') @@ -3579,8 +3560,7 @@ def test_get_easyblock_class_name(self): def test_copy_easyblocks(self): """Test for copy_easyblocks function.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # easybuild/easyblocks subdirectory must exist in target directory error_pattern = "Could not find easybuild/easyblocks subdir in .*" @@ -3654,7 +3634,6 @@ def test_copy_framework_files(self): # create empty test/framework/modules.py, to check whether 'new' is set correctly in result ft.write_file(os.path.join(target_dir, 'test', 'framework', 'modules.py'), '') - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_files = [ os.path.join('easybuild', 'tools', 'filetools.py'), os.path.join('test', 'framework', 'modules.py'), @@ -3670,7 +3649,7 @@ def test_copy_framework_files(self): # setup.py is an important test case, since it has no parent directory # (it's straight in the easybuild-framework directory) setup_py = 'setup.py' - if os.path.exists(os.path.join(topdir, setup_py)): + if os.path.exists(os.path.join(REPO_ROOT, setup_py)): test_files.append(os.path.join(setup_py)) expected_entries.append(setup_py) expected_new.append(True) @@ -3679,7 +3658,7 @@ def test_copy_framework_files(self): # so we need to make sure that's the case here as well (may not be in workspace dir on Travis from example) framework_dir = os.path.join(self.test_prefix, 'easybuild-framework') for test_file in test_files: - ft.copy_file(os.path.join(topdir, test_file), os.path.join(framework_dir, test_file)) + ft.copy_file(os.path.join(REPO_ROOT, test_file), os.path.join(framework_dir, test_file)) test_paths = [os.path.join(framework_dir, f) for f in test_files] @@ -3690,7 +3669,7 @@ def test_copy_framework_files(self): self.assertEqual(sorted(res.keys()), ['new', 'paths_in_repo']) for idx, test_file in enumerate(test_files): - orig_path = os.path.join(topdir, test_file) + orig_path = os.path.join(REPO_ROOT, test_file) copied_path = os.path.join(target_dir, test_file) self.assertExists(copied_path) diff --git a/test/framework/robot.py b/test/framework/robot.py index 326e97cbe5..413ebae8c4 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -35,6 +35,7 @@ import tempfile from copy import deepcopy from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from unittest import TextTestRunner import easybuild.framework.easyconfig.easyconfig as ecec @@ -137,7 +138,7 @@ def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ self.install_mock_module() - base_easyconfig_dir = find_full_path(os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')) + base_easyconfig_dir = find_full_path(TEST_ECS_DIR) self.assertTrue(base_easyconfig_dir) easyconfig = { @@ -403,7 +404,6 @@ def test_resolve_dependencies_minimal(self): # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') self.install_mock_module() init_config(build_options={ @@ -411,7 +411,7 @@ def test_resolve_dependencies_minimal(self): 'minimal_toolchains': True, 'use_existing_modules': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) @@ -509,14 +509,14 @@ def test_resolve_dependencies_minimal(self): 'minimal_toolchains': True, 'add_system_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) - impi_txt = read_file(os.path.join(test_easyconfigs, 'i', 'impi', 'impi-5.1.2.150.eb')) + impi_txt = read_file(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertTrue(re.search("^toolchain = SYSTEM", impi_txt, re.M)) - gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb')) + gzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb')) self.assertTrue(re.search("^toolchain = SYSTEM", gzip_txt, re.M)) barec = os.path.join(self.test_prefix, 'bar-1.2.3-foss-2018a.eb') @@ -551,8 +551,7 @@ def test_resolve_dependencies_missing(self): self.install_mock_module() MockModule.avail_modules = [] - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - init_config(build_options={'robot_path': [test_easyconfigs, self.test_prefix]}) + init_config(build_options={'robot_path': [str(TEST_ECS_DIR), self.test_prefix]}) ec = { 'ec': { @@ -615,21 +614,19 @@ def test_det_easyconfig_paths(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) # copy hwloc easyconfig to h/hwloc subdir in robot search path, # to trigger bug fixed in det_easyconfig_paths (.extend rather than .append for '__archive'__ to ignore_subdirs) hwloc_ec = 'hwloc-1.11.8-GCC-6.4.0-2.28.eb' subdir_hwloc = os.path.join(self.test_prefix, 'h', 'hwloc') mkdir(subdir_hwloc, parents=True) - shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) - shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', hwloc_ec), subdir_hwloc) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertNotExists(test_ec) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + TOY_EC, test_ec, # relative path, should be resolved via robot search path hwloc_ec, '--dry-run', @@ -643,7 +640,7 @@ def test_det_easyconfig_paths(self): outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ - (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (TEST_ECS_DIR, 'toy/0.0'), # specified easyconfigs, available at given location (self.test_prefix, 'intel/2018a'), # dependency, found in robot search path (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path (self.test_prefix, 'hwloc/1.11.8-GCC-6.4.0-2.28'), # specified easyconfig, found in robot search path @@ -670,9 +667,9 @@ def test_det_easyconfig_paths(self): self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + TOY_EC, + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), '--dry-run', '--robot', '--tmpdir=%s' % self.test_prefix, @@ -691,10 +688,8 @@ def test_search_paths(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) self.assertNotExists(test_ec) args = [ @@ -715,11 +710,9 @@ def test_github_det_easyconfig_paths_from_commit(self): """Test det_easyconfig_paths function in combination with --from-commit.""" # note: --from-commit does not involve using GitHub API, so no GitHub token required - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - commit = '589282cf52609067616fc2a522f8e4b81f809cb7' args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path + TOY_EC, # absolute path 'toy-0.0-iter.eb', # relative path, available via robot search path # commit in which ReFrame-4.3.2.eb was added, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits @@ -727,7 +720,7 @@ def test_github_det_easyconfig_paths_from_commit(self): 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path '--dry-run', '--robot', - '--robot=%s' % test_ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--unittest-file=%s' % self.logfile, '--tmpdir=%s' % self.test_prefix, ] @@ -760,11 +753,9 @@ def test_github_det_easyconfig_paths_from_pr(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) - shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertNotExists(test_ec) gompi_2018b_txt = '\n'.join([ @@ -779,7 +770,7 @@ def test_github_det_easyconfig_paths_from_pr(self): write_file(os.path.join(self.test_prefix, 'gompi-2018b-test.eb'), gompi_2018b_txt) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + TOY_EC, test_ec, # relative path, should be resolved via robot search path # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227 '--from-pr=22227', @@ -815,10 +806,9 @@ def test_github_det_easyconfig_paths_from_pr(self): def test_get_toolchain_hierarchy(self): """Test get_toolchain_hierarchy function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) fosscuda_hierarchy = get_toolchain_hierarchy({'name': 'fosscuda', 'version': '2018a'}) @@ -849,7 +839,7 @@ def test_get_toolchain_hierarchy(self): # test also --try-toolchain* case, where we want more detailed information init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() @@ -965,7 +955,7 @@ def test_get_toolchain_hierarchy(self): init_config(build_options={ 'add_system_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() @@ -997,7 +987,7 @@ def test_get_toolchain_hierarchy(self): build_options = { 'add_system_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -1019,13 +1009,13 @@ def test_get_toolchain_hierarchy(self): # put faulty foss easyconfig in place to test error reporting broken_gompi = os.path.join(self.test_prefix, 'gompi-2018a.eb') - copy_file(os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb'), broken_gompi) + copy_file(os.path.join(TEST_ECS_DIR, 'g', 'gompi', 'gompi-2018a.eb'), broken_gompi) ectxt = read_file(broken_gompi) ectxt += "\ndependencies += [('GCC', '4.6.4')]" write_file(broken_gompi, ectxt) init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': [self.test_prefix, test_easyconfigs], + 'robot_path': [self.test_prefix, TEST_ECS_DIR], }) tc = {'name': 'gompi', 'version': '2018a'} error_msg = "Multiple versions of GCC found in dependencies of toolchain gompi: 4.6.4, 6.4.0-2.28" @@ -1116,11 +1106,9 @@ def test_tweak_robotpath(self): """Test that the robot correctly resolves the dependencies of tweaked easyconfigs. Tweaked easyconfigs take priority, but tweaked dependencies are only used on an as-needed basis""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # Create directories to store the tweaked easyconfigs tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) + robot_path = det_robot_path([TEST_ECS_DIR], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), @@ -1129,7 +1117,7 @@ def test_tweak_robotpath(self): }) # Parse the easyconfig that we want to tweak - untweaked_openmpi = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi, False)]) # Tweak the toolchain version of the easyconfig @@ -1149,7 +1137,7 @@ def test_tweak_robotpath(self): # Check it picks up the tweaked OpenMPI self.assertIn(tweaked_openmpi, specs) # Check it picks up the untweaked dependency of the tweaked OpenMPI - untweaked_hwloc = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb') + untweaked_hwloc = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb') self.assertIn(untweaked_hwloc, specs) # Check correctness of tweak_map (maps back to the original untweaked file, even for hwloc, where the # tweaked version is generated but not used) @@ -1159,8 +1147,7 @@ def test_tweak_robotpath(self): def test_robot_find_subtoolchain_for_dep(self): """Test robot_find_subtoolchain_for_dep.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - init_config(build_options={'robot_path': test_easyconfigs}) + init_config(build_options={'robot_path': TEST_ECS_DIR}) # # First test that it can do basic resolution @@ -1192,7 +1179,7 @@ def test_robot_find_subtoolchain_for_dep(self): # init_config(build_options={ 'add_system_to_minimal_toolchains': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) # specify alternative parent toolchain gompi_1410 = {'name': 'gompi', 'version': '2018a'} @@ -1244,7 +1231,7 @@ def test_robot_find_subtoolchain_for_dep(self): write_file(barec, barec_txt) # check without --minimal-toolchains - init_config(build_options={'robot_path': test_easyconfigs}) + init_config(build_options={'robot_path': TEST_ECS_DIR}) bar = EasyConfig(barec) expected_dep_versions = { @@ -1260,7 +1247,7 @@ def test_robot_find_subtoolchain_for_dep(self): # check with --minimal-toolchains enabled init_config(build_options={ 'minimal_toolchains': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) bar = EasyConfig(barec) @@ -1294,7 +1281,7 @@ def test_robot_find_subtoolchain_for_dep(self): init_config(build_options={ 'minimal_toolchains': True, 'use_existing_modules': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) # Check gompi is now being picked up @@ -1315,10 +1302,9 @@ def test_robot_find_subtoolchain_for_dep_ecs_vs_mods(self): Test behaviour of robot_find_subtoolchain_for_dep w.r.t. picking subtoolchains based on easyconfigs vs modules. """ - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # include both test easyconfig files and test directory in robot search path - build_options = {'robot_path': [test_easyconfigs, self.test_prefix]} + build_options = {'robot_path': [str(TEST_ECS_DIR), self.test_prefix]} init_config(build_options=build_options) test_mods_dir = os.path.join(self.test_prefix, 'modules') @@ -1399,17 +1385,16 @@ def test_robot_find_subtoolchain_for_dep_ecs_vs_mods(self): def test_check_conflicts(self): """Test check_conflicts function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'force': True, 'retain_all_deps': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) - gzip_ec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') - gompi_ec = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + gompi_ec = os.path.join(TEST_ECS_DIR, 'g', 'gompi', 'gompi-2018a.eb') non_conflict_ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)]) # no conflicts found, no output to stderr @@ -1436,7 +1421,7 @@ def test_check_conflicts(self): self.assertIn("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28", stderr) # Can also return the text - with self.mocked_stdout_stderr(mock_stdout=False) as mocked_stderr: + with self.mocked_stderr() as mocked_stderr: conflict_lst = check_conflicts(ecs, self.modtool, return_conflicts=True) self.assertEqual('\n'.join(conflict_lst), stderr.strip()) self.assertEqual(mocked_stderr.getvalue(), '') @@ -1446,8 +1431,8 @@ def test_check_conflicts(self): # direct conflict on software version ecs, _ = parse_easyconfigs([ - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-6.4.0-2.28.eb'), False), - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), + (os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-6.4.0-2.28.eb'), False), + (os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), ]) with self.mocked_stderr(): conflicts = check_conflicts(ecs, self.modtool) @@ -1458,8 +1443,8 @@ def test_check_conflicts(self): # indirect conflict on dependencies ecs, _ = parse_easyconfigs([ - (os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), False), - (os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), False), + (os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), False), + (os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), False), ]) with self.mocked_stderr(): conflicts = check_conflicts(ecs, self.modtool) @@ -1472,9 +1457,9 @@ def test_check_conflicts(self): self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found") # Conflict in build dependencies is fine - hwloc_txt = read_file(os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) - gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) - bzip_txt = read_file(os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) + hwloc_txt = read_file(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + gzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + bzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) tc = re.search(r"toolchain *=.*", hwloc_txt)[0] bzip_txt += f"\n{tc}" gzip_txt += f"\n{tc}" @@ -1495,9 +1480,6 @@ def test_check_conflicts(self): def test_check_conflicts_wrapper_deps(self): """Test check_conflicts when dependency 'wrappers' are involved.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - wrapper_ec_txt = '\n'.join([ "easyblock = 'ModuleRC'", "name = 'toy'", @@ -1510,7 +1492,7 @@ def test_check_conflicts_wrapper_deps(self): wrapper_ec = os.path.join(self.test_prefix, 'toy-0.eb') write_file(wrapper_ec, wrapper_ec_txt) - ecs, _ = parse_easyconfigs([(toy_ec, False), (wrapper_ec, False)]) + ecs, _ = parse_easyconfigs([(TOY_EC, False), (wrapper_ec, False)]) with self.mocked_stderr(): res = check_conflicts(ecs, self.modtool) stderr = self.get_stderr() @@ -1520,12 +1502,9 @@ def test_check_conflicts_wrapper_deps(self): def test_check_conflicts_multi_deps(self): """Test check_conflicts when multi_deps is used.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT tc_regex = re.compile(r'^toolchain = .*', re.M) test_ec_txt = tc_regex.sub("toolchain = SYSTEM", test_ec_txt) test_ec_txt += "\nmulti_deps = {'GCC': ['4.9.2', '7.3.0-2.30']}\n" @@ -1534,7 +1513,7 @@ def test_check_conflicts_multi_deps(self): write_file(test_ec, test_ec_txt) ecs, _ = parse_easyconfigs([(test_ec, False)]) - init_config(build_options={'robot_path': [test_ecs]}) + init_config(build_options={'robot_path': [TEST_ECS_DIR]}) # use of multi_deps should not result in false positives in check_conflicts self.assertFalse(check_conflicts(ecs, self.modtool)) @@ -1546,9 +1525,7 @@ def test_robot_archived_easyconfigs(self): self.allow_deprecated_behaviour() init_config(build_options={'silent': True}) - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-intel-2018a.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-intel-2018a.eb') gzip_ectxt = read_file(gzip_ec) test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1563,29 +1540,28 @@ def test_robot_archived_easyconfigs(self): # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs init_config(build_options={ 'consider_archived_easyconfigs': True, - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, }) res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) self.assertEqual([ec['full_mod_name'] for ec in res], ['intel/2012a', 'gzip/1.5-intel-2012a']) - expected = os.path.join(test_ecs, '__archive__', 'i', 'intel', 'intel-2012a.eb') + expected = os.path.join(TEST_ECS_DIR, '__archive__', 'i', 'intel', 'intel-2012a.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected)) def test_search_easyconfigs(self): """Test search_easyconfigs function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'search_paths': [self.test_prefix], }) # copy some files to search_paths location - copy_file(os.path.join(test_ecs, 'b', 'binutils', 'binutils-2.25-GCCcore-4.9.3.eb'), self.test_prefix) - copy_file(os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), self.test_prefix) + copy_file(os.path.join(TEST_ECS_DIR, 'b', 'binutils', 'binutils-2.25-GCCcore-4.9.3.eb'), self.test_prefix) + copy_file(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), self.test_prefix) paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', consider_extra_paths=False, print_result=False) - ref_paths = [os.path.join(test_ecs, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb']] + ref_paths = [os.path.join(TEST_ECS_DIR, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb']] self.assertEqual(len(paths), 2) self.assertEqual(paths, ref_paths) @@ -1598,17 +1574,17 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), + os.path.join(TEST_ECS_DIR, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') ] self.assertEqual(paths, ref_paths) # now do a case sensitive search paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False, case_sensitive=True) - ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] + ref_paths = [os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] self.assertEqual(paths, ref_paths) # test use of filename_only @@ -1631,7 +1607,7 @@ def test_search_easyconfigs(self): if filename_only: path = ec_fn else: - path = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs', 'b', 'binutils', ec_fn) + path = os.path.join(TEST_ECS_DIR, 'b', 'binutils', ec_fn) pattern.append(r"^ \* .*%s$" % path) regex = re.compile('\n'.join(pattern), re.M) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index b698ea013a..dde290348e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,6 +38,7 @@ import unittest from contextlib import contextmanager from importlib import reload +from pathlib import Path from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -316,7 +317,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal # always run main in unit testing mode (which for example allows for using deprecated toolchains); # note: don't change 'args' value, which is passed by reference! - main_args = args + ['--unit-testing-mode'] + main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode'] myerr = False if logfile is None: