diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..120eb9640a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, str): + if isinstance(paths, (str, Path)): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/test/framework/__init__.py b/test/framework/__init__.py index 895b15e622..04daadf7fa 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -27,3 +27,13 @@ @author: Toon Willems (Ghent University) """ + +from pathlib import Path +from easybuild.tools.filetools import read_file + +TEST_DIR = Path(__file__).parent +REPO_ROOT = TEST_DIR.parent.parent +TEST_MODULES_DIR = TEST_DIR / 'modules' +TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs' +TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb' +TOY_EC_TXT: str = read_file(TOY_EC) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 5834fb731c..ba7cba5833 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -45,6 +45,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod from easybuild.tools.utilities import quote_str +from test.framework import TEST_MODULES_DIR, TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config @@ -57,8 +58,7 @@ def setUp(self): """Test setup.""" super().setUp() # find .eb file - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + eb_path = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') eb_full_path = find_full_path(eb_path) self.assertTrue(eb_full_path) @@ -801,9 +801,8 @@ def test_module_extensions(self): # check if extensions option is enabled and some module extensions are defined init_config(build_options={'module_extensions': True}) - test_dir = os.path.abspath(os.path.dirname(__file__)) - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR) + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -833,7 +832,7 @@ def test_module_extensions(self): self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc)) # check if the extensions is missing if there are no extensions - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -849,7 +848,7 @@ def test_module_extensions(self): # check if the extensions is missing if 'module_extensions' is disabled init_config(build_options={'module_extensions': False}) - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -1358,15 +1357,14 @@ def test_module_naming_scheme(self): all_stops = [x[0] for x in EasyBlock.get_steps()] init_config(build_options={'valid_stops': all_stops}) - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files] + ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(TEST_ECS_DIR) for fil in files] # keep only easyconfig files (there may be additional files like patches, checksums.json, etc.) ec_files = [x for x in ec_files if x.endswith('.eb')] build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1421,7 +1419,7 @@ def test_mns(): init_config(build_options=build_options) err_pattern = 'nosucheasyconfigparameteravailable' - ec_file = os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') self.assertErrorRegex(EasyBuildError, err_pattern, EasyConfig, ec_file) # test simple custom module naming scheme @@ -1438,7 +1436,7 @@ def test_mns(): } test_mns() - ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/2018a') # test module naming scheme using all available easyconfig parameters @@ -1485,7 +1483,7 @@ def test_mns(): # determine full module name self.assertEqual(ActiveMNS().det_full_module_name(dep_spec), ec2mod_map[dep_ec]) - ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True) self.assertEqual(ec.full_mod_name, ec2mod_map['gzip-1.5-foss-2018a.eb']) self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/e69469ac250145c9e814e5dde93f5fde6d80375d') @@ -1554,11 +1552,10 @@ def test_hierarchical_mns(self): """Test hierarchical module naming scheme.""" moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'] - ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, 'valid_module_classes': moduleclasses, @@ -1566,7 +1563,7 @@ def test_hierarchical_mns(self): def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): """Test whether active module naming scheme returns expected values.""" - ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0]) + ec = EasyConfig(glob.glob(os.path.join(TEST_ECS_DIR, '*', '*', ecfile))[0]) self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) @@ -1652,7 +1649,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, test_ec(ecfile, *mns_vals) # impi with dummy toolchain, which doesn't make sense in a hierarchical context - ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' @@ -1695,7 +1692,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, test_ec(ecfile, *mns_vals, init_modpaths=['Core/%s' % c for c in moduleclasses]) # impi with dummy toolchain, which doesn't make sense in a hierarchical context - ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedModuleNamingScheme' diff --git a/test/framework/modules.py b/test/framework/modules.py index 589df6ccd5..f257fa7c40 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -36,6 +36,7 @@ import shutil import stat import sys +from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -64,7 +65,7 @@ class ModulesTest(EnhancedTestCase): def init_testmods(self, test_modules_paths=None): """Initialize set of test modules for test.""" if test_modules_paths is None: - test_modules_paths = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))] + test_modules_paths = [os.path.abspath(os.path.join(TEST_MODULES_DIR))] self.reset_modulepath(test_modules_paths) # for Lmod, this test has to run first, to avoid that it fails; @@ -83,7 +84,7 @@ def test_long_module_path(self): # copy one of the test modules there gcc_mod_dir = os.path.join(long_mod_path, 'GCC') os.makedirs(gcc_mod_dir) - gcc_mod_path = os.path.join(os.path.dirname(__file__), 'modules', 'GCC', '4.6.3') + gcc_mod_path = os.path.join(TEST_MODULES_DIR, 'GCC', '4.6.3') copy_file(gcc_mod_path, gcc_mod_dir) # try and use long modules path @@ -97,8 +98,6 @@ def test_long_module_path(self): def test_run_module(self): """Test for ModulesTool.run_module method.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']: os.environ.pop(key, None) @@ -107,7 +106,7 @@ def test_run_module(self): self.modtool.run_module('load', 'GCC/6.4.0-2.28') self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') - self.reset_modulepath([os.path.join(testdir, 'modules')]) + self.reset_modulepath([os.path.join(TEST_MODULES_DIR)]) self.assertNotIn('EBROOTGCC', os.environ) self.modtool.run_module(['load', 'GCC/6.4.0-2.28']) @@ -266,7 +265,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['OpenMPI'], maybe_partial=False, skip_avail=True), [False]) # exist works on hidden modules in Lua syntax (only with Lmod) - test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + test_modules_path = os.path.join(TEST_MODULES_DIR) if isinstance(self.modtool, Lmod): # make sure only the .lua module file is there, otherwise this test doesn't work as intended self.assertExists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua')) @@ -626,7 +625,7 @@ def test_prepend_module_path(self): self.assertEqual(modulepath, curr_module_paths()) # prepending path that is 'deeper down' in $MODULEPATH works, brings it back to front - test_mods_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_mods_dir = os.path.join(TEST_MODULES_DIR) self.assertTrue(any(os.path.samefile(test_mods_dir, p) for p in modulepath)) self.modtool.prepend_module_path(test_mods_dir) self.assertTrue(os.path.samefile(curr_module_paths()[0], test_mods_dir)) @@ -771,7 +770,7 @@ def check_get_software_libdir(expected, **additional_args): def test_wrong_modulepath(self): """Test whether modules tool can deal with a broken $MODULEPATH.""" - test_modules_path = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) + test_modules_path = os.path.realpath(os.path.join(TEST_MODULES_DIR)) modules_test_installpath = os.path.join(self.test_installpath, 'modules', 'all') os.environ['MODULEPATH'] = '/some/non-existing/path:/this/doesnt/exists/anywhere:%s' % test_modules_path init_config() @@ -785,8 +784,7 @@ def test_wrong_modulepath(self): def test_modulefile_path(self): """Test modulefile_path method""" - test_dir = os.path.abspath(os.path.dirname(__file__)) - gcc_mod_file = os.path.join(test_dir, 'modules', 'GCC', '6.4.0-2.28') + gcc_mod_file = os.path.join(TEST_MODULES_DIR, 'GCC', '6.4.0-2.28') modtool = modules_tool() res = modtool.modulefile_path('GCC/6.4.0-2.28') @@ -794,7 +792,7 @@ def test_modulefile_path(self): if isinstance(self.modtool, Lmod): res = modtool.modulefile_path('bzip2/.1.0.6') - self.assertTrue(os.path.samefile(res, os.path.join(test_dir, 'modules', 'bzip2', '.1.0.6.lua'))) + self.assertTrue(os.path.samefile(res, os.path.join(TEST_MODULES_DIR, 'bzip2', '.1.0.6.lua'))) res = modtool.modulefile_path('bzip2/.1.0.6', strip_ext=True) self.assertTrue(res.endswith('test/framework/modules/bzip2/.1.0.6')) @@ -814,11 +812,10 @@ def test_path_to_top_of_module_tree(self): def test_path_to_top_of_module_tree_hierarchical_mns(self): """Test function to determine path to top of the module tree for a hierarchical module naming scheme.""" - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -853,7 +850,7 @@ def test_path_to_top_of_module_tree_lua(self): """Test path_to_top_of_module_tree function on modules in Lua syntax.""" if isinstance(self.modtool, Lmod): orig_modulepath = os.environ.get('MODULEPATH') - self.modtool.unuse(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) + self.modtool.unuse(os.path.join(TEST_MODULES_DIR)) curr_modulepath = os.environ.get('MODULEPATH') error_msg = "Incorrect $MODULEPATH value after unuse: %s (orig: %s)" % (curr_modulepath, orig_modulepath) self.assertEqual(curr_modulepath, None, error_msg) @@ -1024,11 +1021,10 @@ def test_path_to_top_of_module_tree_categorized_hmns(self): scheme. """ - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1063,7 +1059,7 @@ def test_path_to_top_of_module_tree_categorized_hmns(self): def test_modules_tool_stateless(self): """Check whether ModulesTool instance is stateless between runs.""" - test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_modules_path = os.path.join(TEST_MODULES_DIR) # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load copy_dir(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) @@ -1159,7 +1155,7 @@ def test_module_caches(self): # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH # invalidate_module_caches_for should be able to deal with this - test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_mods_path = os.path.join(TEST_MODULES_DIR) mods_symlink = os.path.join(self.test_prefix, 'modules_symlink') os.symlink(test_mods_path, mods_symlink) self.modtool.use(mods_symlink) @@ -1588,11 +1584,8 @@ def test_get_setenv_value_from_modulefile(self): init_config(build_options={'generate_devel_module': True}) - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(eb_path)) + write_file(test_ec, TOY_EC_TXT) write_file(test_ec, "\nmodextravars = {'FOO': 'value with spaces'}", append=True) toy_eb = EasyBlock(EasyConfig(test_ec)) diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 9022fd3953..acfcd45533 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -31,6 +31,7 @@ import re import stat import sys +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -38,7 +39,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, update_build_option -from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_dir, which, write_file +from easybuild.tools.filetools import adjust_permissions, mkdir, remove_dir, which, write_file from easybuild.tools.job import pbs_python from easybuild.tools.job.pbs_python import PbsPython from easybuild.tools.options import parse_options @@ -128,18 +129,16 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob - topdir = os.path.dirname(os.path.abspath(__file__)) - build_options = { 'external_modules_metadata': {}, - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) @@ -148,7 +147,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) @@ -182,8 +181,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): self.assertIn('GCC-4.6.3.eb', jobs[3].deps[1].script) # also test use of --pre-create-installdir - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) + ordered_ecs = resolve_dependencies(process_easyconfig(TOY_EC), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') @@ -242,27 +240,23 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) - topdir = os.path.dirname(os.path.abspath(__file__)) - build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - easyconfigs = process_easyconfig(ec_file) + easyconfigs = process_easyconfig(TOY_EC) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') - cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) + test_easyblocks_path = os.path.join(TEST_DIR, 'sandbox') + cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (REPO_ROOT, test_easyblocks_path) with self.mocked_stdout_stderr(): build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) @@ -275,10 +269,9 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): # also check what happens when a job fails (an error should be raised) test_ecfile = os.path.join(self.test_prefix, 'test.eb') - ectxt = read_file(ec_file) # use different version, for which no sources are available regex = re.compile('^version = .*', re.M) - ectxt = regex.sub("version = '1.2.3'", ectxt) + ectxt = regex.sub("version = '1.2.3'", TOY_EC_TXT) write_file(test_ecfile, ectxt) ecs = resolve_dependencies(process_easyconfig(test_ecfile), self.modtool) @@ -288,8 +281,6 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): def test_submit_jobs(self): """Test submit_jobs""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = process_easyconfig(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')) args = [ '--debug', @@ -302,7 +293,7 @@ def test_submit_jobs(self): '--job-cores=3', ] eb_go = parse_options(args=args) - cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True) + cmd = submit_jobs(process_easyconfig(TOY_EC), eb_go.generate_cmd_line(), testing=True) # these patterns must be found regexs = [ @@ -330,7 +321,7 @@ def test_submit_jobs(self): # test again with custom EasyBuild command to use in jobs update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug") - cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True) + cmd = submit_jobs(process_easyconfig(TOY_EC), eb_go.generate_cmd_line(), testing=True) regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ") self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd)) @@ -348,13 +339,12 @@ def test_build_easyconfigs_in_parallel_slurm(self): os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') - foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') + test_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + foss_ec = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, diff --git a/test/framework/tweak.py b/test/framework/tweak.py index b59fbd4557..eb4dfe3f4d 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -29,6 +29,7 @@ """ import os import sys +from test.framework import TEST_ECS_DIR, TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -74,12 +75,11 @@ def test_pick_version(self): def test_find_matching_easyconfigs(self): """Test find_matching_easyconfigs function.""" - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') for (name, installver) in [('GCC', '4.8.2'), ('gzip', '1.5-foss-2018a')]: - ecs = find_matching_easyconfigs(name, installver, [test_easyconfigs_path]) + ecs = find_matching_easyconfigs(name, installver, [TEST_ECS_DIR]) self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver))) - ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path]) + ecs = find_matching_easyconfigs('GCC', '*', [TEST_ECS_DIR]) gccvers = ['10.2.0', '12.3.0', '4.6.3', '4.6.4', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25', '4.9.3-2.26', '6.4.0-2.28', '7.3.0-2.30'] self.assertEqual(len(ecs), len(gccvers)) @@ -92,14 +92,13 @@ def test_obtain_ec_for(self): """Test obtain_ec_for function.""" init_config(build_options={'silent': True}) - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # find existing easyconfigs specs = { 'name': 'GCC', 'version': '6.4.0', 'versionsuffix': '-2.28', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-6.4.0-2.28.eb') @@ -110,7 +109,7 @@ def test_obtain_ec_for(self): 'toolchain_version': '2018a', 'versionsuffix': '-OpenBLAS-0.2.20', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb') @@ -118,7 +117,7 @@ def test_obtain_ec_for(self): 'name': 'ifort', 'versionsuffix': '-GCC-4.9.3-2.25', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'ifort-2016.1.150-GCC-4.9.3-2.25.eb') @@ -126,7 +125,7 @@ def test_obtain_ec_for(self): specs = { 'name': 'GCC', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-12.3.0.eb') @@ -136,48 +135,45 @@ def test_obtain_ec_for(self): 'name': 'GCC', 'version': '4.9.0', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertTrue(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.0.eb') def test_tweak_one_version(self): """Test tweak_one function""" - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb') # test tweaking of software version (--try-software-version) - tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb') - tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + tweaked_TOY_EC = os.path.join(self.test_prefix, 'toy-tweaked.eb') + tweak_one(TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) - toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict() - tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() + TOY_EC_parsed = EasyConfigParser(TOY_EC).get_config_dict() + tweaked_TOY_EC_parsed = EasyConfigParser(tweaked_TOY_EC).get_config_dict() # checksums should be reset to empty list, only version should be changed, nothing else - self.assertEqual(tweaked_toy_ec_parsed['checksums'], []) - self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') - for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: - val = toy_ec_parsed[key] - self.assertIn(key, tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) - tweaked_val = tweaked_toy_ec_parsed.get(key) + self.assertEqual(tweaked_TOY_EC_parsed['checksums'], []) + self.assertEqual(tweaked_TOY_EC_parsed['version'], '1.2.3') + for key in [k for k in TOY_EC_parsed.keys() if k not in ['checksums', 'version']]: + val = TOY_EC_parsed[key] + self.assertIn(key, tweaked_TOY_EC_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) + tweaked_val = tweaked_TOY_EC_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) # check behaviour if target file already exists error_pattern = "File exists, not overwriting it without --force" - self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) # existing file does get overwritten when --force is used init_config(build_options={'force': True, 'silent': True}) - write_file(tweaked_toy_ec, '') - tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) - tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() - self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') + write_file(tweaked_TOY_EC, '') + tweak_one(TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) + tweaked_TOY_EC_parsed = EasyConfigParser(tweaked_TOY_EC).get_config_dict() + self.assertEqual(tweaked_TOY_EC_parsed['version'], '1.2.3') def test_check_capability_mapping(self): """Test comparing the functionality of two toolchains""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True) @@ -206,9 +202,8 @@ def test_check_capability_mapping(self): def test_match_minimum_tc_specs(self): """Test matching a toolchain to lowest possible in a hierarchy""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': module_classes(), }) @@ -244,10 +239,9 @@ def test_match_minimum_tc_specs(self): def test_dep_tree_of_toolchain(self): """Test getting list of dependencies of a toolchain (as EasyConfig objects)""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'check_osdeps': False, }) toolchain_spec = {'name': 'foss', 'version': '2018a'} @@ -267,9 +261,8 @@ def test_dep_tree_of_toolchain(self): def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': module_classes(), }) @@ -311,16 +304,15 @@ def test_map_toolchain_hierarchies(self): def test_get_matching_easyconfig_candidates(self): """Test searching for easyconfig candidates based on a stub and toolchain""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], }) toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} paths, toolchain_suff = get_matching_easyconfig_candidates('gzip-', toolchain) expected_toolchain_suff = '-GCC-4.9.3-2.26' self.assertEqual(toolchain_suff, expected_toolchain_suff) - expected_paths = [os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_suff + '.eb')] + expected_paths = [os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_suff + '.eb')] self.assertEqual(paths, expected_paths) paths, toolchain_stub = get_matching_easyconfig_candidates('nosuchmatch', toolchain) @@ -329,9 +321,8 @@ def test_get_matching_easyconfig_candidates(self): def test_map_common_versionsuffixes(self): """Test mapping between two toolchain hierarchies""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), }) @@ -355,9 +346,8 @@ def test_map_common_versionsuffixes(self): def test_find_potential_version_mappings(self): """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), }) @@ -367,7 +357,7 @@ def test_find_potential_version_mappings(self): iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) - ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') parsed_ec = process_easyconfig(ec_spec)[0] gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'][0] self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26') @@ -376,7 +366,7 @@ def test_find_potential_version_mappings(self): self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain expected = { - 'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), + 'path': os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), 'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'version': '1.6', 'versionsuffix': '', @@ -408,7 +398,7 @@ def test_find_potential_version_mappings(self): potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True) self.assertEqual(len(potential_versions), 1) expected = { - 'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'), + 'path': os.path.join(TEST_ECS_DIR, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'), 'toolchain': {'version': '6.4.0-2.28', 'name': 'GCC'}, 'version': '0.2.20', 'versionsuffix': '', @@ -417,9 +407,8 @@ def test_find_potential_version_mappings(self): def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } @@ -430,7 +419,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) - ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() @@ -497,7 +486,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): # Check that if we update a software version, it also updates the version if the software appears in an # extension list (like for a PythonBundle) - ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # Create the trivial toolchain mapping toolchain = {'name': 'gompi', 'version': '2018a'} tc_mapping = map_toolchain_hierarchies(toolchain, toolchain, self.modtool) @@ -523,20 +512,19 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): def test_list_deps_versionsuffixes(self): """Test listing of dependencies' version suffixes""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) get_toolchain_hierarchy.clear() - ec_spec = os.path.join(test_easyconfigs, 'g', 'golf', 'golf-2018a.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'g', 'golf', 'golf-2018a.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), ['-serial']) - ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), []) - ec_spec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), ['-deps']) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index b698ea013a..dde290348e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,6 +38,7 @@ import unittest from contextlib import contextmanager from importlib import reload +from pathlib import Path from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -316,7 +317,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal # always run main in unit testing mode (which for example allows for using deprecated toolchains); # note: don't change 'args' value, which is passed by reference! - main_args = args + ['--unit-testing-mode'] + main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode'] myerr = False if logfile is None: