diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..120eb9640a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, str): + if isinstance(paths, (str, Path)): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/test/framework/__init__.py b/test/framework/__init__.py index 895b15e622..04daadf7fa 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -27,3 +27,13 @@ @author: Toon Willems (Ghent University) """ + +from pathlib import Path +from easybuild.tools.filetools import read_file + +TEST_DIR = Path(__file__).parent +REPO_ROOT = TEST_DIR.parent.parent +TEST_MODULES_DIR = TEST_DIR / 'modules' +TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs' +TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb' +TOY_EC_TXT: str = read_file(TOY_EC) diff --git a/test/framework/options.py b/test/framework/options.py index 8c9c037ef0..1d02bd49a8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -66,6 +66,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type from easybuild.tools.version import VERSION +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config from test.framework.github import ignore_rate_limit_in_pr @@ -250,7 +251,7 @@ def test_force(self): """Test forcing installation even if the module is already available.""" # use GCC-4.6.3.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb') + eb_file = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') # check log message without --force args = [ @@ -279,12 +280,10 @@ def test_force(self): def test_skip(self): """Test skipping installation of module (--skip, -k).""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # check log message with --skip for existing module args = [ - toy_ec, + TOY_EC, '--force', '--debug', ] @@ -305,7 +304,7 @@ def test_skip(self): # check log message with --skip for non-existing module args = [ - toy_ec, + TOY_EC, '--try-software-version=1.2.3.4.5.6.7.8.9', '--try-amend=sources=toy-0.0.tar.gz,toy-0.0.tar.gz', # hackish, but fine '--force', @@ -328,7 +327,7 @@ def test_skip(self): # make sure that sanity check is *NOT* skipped under --skip test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT regex = re.compile(r"sanity_check_paths = \{(.|\n)*\}", re.M) test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt) write_file(test_ec, test_ec_txt) @@ -342,11 +341,9 @@ def test_skip(self): def test_module_only_param(self): """check use of module_only parameter""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\nmodule_only=True\n" test_ec_txt += "\nskipsteps = ['sanitycheck']\n" # Software does not exist, so sanity check would fail write_file(test_ec, test_ec_txt) @@ -369,12 +366,10 @@ def test_module_only_param(self): def test_skipsteps(self): """Test skipping of steps using skipsteps.""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # make sure that sanity check is *NOT* skipped test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT regex = re.compile(r"sanity_check_paths = \{(.|\n)*\}", re.M) test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt) write_file(test_ec, test_ec_txt) @@ -409,8 +404,7 @@ def test_skipsteps(self): def test_skip_test_step(self): """Test skipping testing the build (--skip-test-step).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') # check log message without --skip-test-step args = [ @@ -443,9 +437,8 @@ def test_skip_test_step(self): def test_ignore_test_failure(self): """Test ignore failing tests (--ignore-test-failure).""" - topdir = os.path.abspath(os.path.dirname(__file__)) # This EC uses a `runtest` command which does not exist and hence will make the test step fail - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') args = [toy_ec, '--ignore-test-failure', '--force'] @@ -466,10 +459,8 @@ def test_ignore_test_failure(self): def test_skip_sanity_check(self): """Test skipping of sanity check step (--skip-sanity-check).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec) + "\nsanity_check_commands = ['this_will_fail']") + write_file(test_ec, TOY_EC_TXT + "\nsanity_check_commands = ['this_will_fail']") args = [test_ec, '--rebuild'] err_msg = "Sanity check failed" @@ -491,7 +482,7 @@ def test_job(self): """Test submitting build as a job.""" # use gzip-1.4.eb easyconfig file that comes with the tests - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR eb_file = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb') def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb_file='gzip-1.4.eb'): @@ -535,7 +526,7 @@ def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb # check if libtoy dep uses --try-toolchain but gzip does not (easyconfig exists already) eb_file = os.path.join(self.test_buildpath, 'toy-0.0-with-deps.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), eb_file) + copy_file(TOY_EC, eb_file) write_file(eb_file, "dependencies = [('libtoy', '0.0'), ('gzip', '1.4')]\n", append=True) try_opts = " --try-toolchain='GCC,4.9.3-2.26'" tweaked_eb_file = "toy-0.0-GCC-4.9.3-2.26.eb" @@ -576,8 +567,7 @@ def test_zzz_logtostdout(self): error_msg = "Log messages are printed to stdout when %s is used (stdout: %s)" % (stdout_arg, stdout) self.assertTrue(len(stdout) > 100, error_msg) - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ecfile = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ecfile = TOY_EC self.logfile = None with self.mocked_stdout_stderr(mock_stderr=False): @@ -945,8 +935,7 @@ def test_avail_cfgfile_constants(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + copy_dir(TEST_ECS_DIR, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -1084,15 +1073,9 @@ def test_000_list_easyblocks(self): def test_search(self): """Test searching for easyconfigs.""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - - # simple search - args = [ - '--search=gzip', - '--robot=%s' % test_easyconfigs_dir, - ] + args = ['--robot=%s' % TEST_ECS_DIR] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search=gzip'], testing=False) txt = self.get_stdout() for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]: @@ -1100,12 +1083,8 @@ def test_search(self): self.assertRegex(txt, regex) # search w/ regex - args = [ - '--search=^gcc.*2.eb', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search=^gcc.*2.eb'], testing=False) txt = self.get_stdout() for ec in ['GCC-4.8.2.eb', 'GCC-4.9.2.eb']: @@ -1122,12 +1101,8 @@ def test_search(self): ] # test --search-filename - args = [ - '--search-filename=^gcc', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search-filename=^gcc'], testing=False) txt = self.get_stdout() for ec in gcc_ecs: @@ -1135,13 +1110,8 @@ def test_search(self): self.assertRegex(txt, regex) # test --search-filename --terse - args = [ - '--search-filename=^gcc', - '--terse', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search-filename=^gcc', '--terse'], testing=False) txt = self.get_stdout() for ec in gcc_ecs: @@ -1150,14 +1120,8 @@ def test_search(self): # also test --search-short/-S for search_arg in ['-S', '--search-short']: - args = [ - search_arg, - '^toy-0.0', - '-r', - test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, raise_error=True, verbose=True, testing=False) + self.eb_main(args + [search_arg, '^toy-0.0'], raise_error=True, verbose=True, testing=False) txt = self.get_stdout() self.assertRegex(txt, re.compile(r'^CFGS\d+=', re.M)) @@ -1165,13 +1129,8 @@ def test_search(self): self.assertRegex(txt, r" \* \$CFGS\d+/*%s" % ec) # combining --search with --try-* should not cause trouble; --try-* should just be ignored - args = [ - '--search=^gcc', - '--robot-paths=%s' % test_easyconfigs_dir, - '--try-toolchain-version=1.2.3', - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False, raise_error=True) + self.eb_main(args + ['--search=^gcc', '--try-toolchain-version=1.2.3'], testing=False, raise_error=True) txt = self.get_stdout() self.assertIn('GCC-4.9.2', txt) @@ -1180,9 +1139,8 @@ def test_search(self): # characters like ^, . or * are not touched, since these can be used as regex characters in queries for opt in ['--search', '-S', '--search-short']: for pattern in ['netCDF-C++', 'foo|bar', '^foo', 'foo.*bar']: - args = [opt, pattern, '--robot', test_easyconfigs_dir] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, raise_error=True, verbose=True, testing=False) + self.eb_main(args + [opt, pattern], raise_error=True, verbose=True, testing=False) stdout = self.get_stdout() # there shouldn't be any hits for any of these queries, so empty output... self.assertEqual(stdout.strip(), '') @@ -1192,14 +1150,14 @@ def test_search(self): # a proper error is produced in that case (as opposed to a crash) for opt in ['--search', '-S', '--search-short']: for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']: - args = [opt, pattern, '--robot', test_easyconfigs_dir] with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True) + self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args + [opt, pattern], + raise_error=True) # test searching for non-existing easyconfig file (should produce non-zero exit code) # 4 corresponds with MISSING_EASYCONFIG in EasyBuildExit (see easybuild/tools/build_log.py) - args = ['--search', 'nosuchsoftware-1.2.3.4.5'] - self.assertErrorRegex(SystemExit, 'MISSING_EASYCONFIG|4', self.eb_main, args, + self.assertErrorRegex(SystemExit, 'MISSING_EASYCONFIG|4', self.eb_main, + args + ['--search', 'nosuchsoftware-1.2.3.4.5'], testing=False, raise_error=True, raise_systemexit=True) def test_ignore_index(self): @@ -1207,9 +1165,7 @@ def test_ignore_index(self): Test use of --ignore-index. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') - copy_file(toy_ec, self.test_prefix) + copy_file(TOY_EC, self.test_prefix) toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb'] @@ -1303,11 +1259,8 @@ def mocked_main(self, args, **kwargs): def test_copy_ec(self): """Test --copy-ec.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') - - toy_ec_txt = read_file(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')) - bzip2_ec_txt = read_file(os.path.join(test_easyconfigs_dir, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) + toy_ec_txt = TOY_EC_TXT + bzip2_ec_txt = read_file(os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1732,8 +1685,7 @@ def test_dry_run_short(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + copy_dir(TEST_ECS_DIR, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -1779,7 +1731,7 @@ def test_try_robot_force(self): os.close(fd) # use toy-0.0.eb easyconfig file that comes with the tests - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR eb1 = os.path.join(test_ecs, 'f', 'FFTW', 'FFTW-3.3.7-gompi-2018a.eb') eb2 = os.path.join(test_ecs, 's', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb') @@ -1815,7 +1767,7 @@ def test_try_robot_force(self): def test_try_toolchain_mapping(self): """Test mapping of subtoolchains with --try-toolchain.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') args = [ @@ -2043,7 +1995,7 @@ def test_github_from_pr(self): '--from-pr=22227', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -2072,7 +2024,7 @@ def test_github_from_pr(self): '--from-pr=22227,19834', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -2112,7 +2064,7 @@ def test_github_from_pr_token_log(self): '--dry-run', '--debug', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user ] try: @@ -2137,7 +2089,7 @@ def test_github_from_pr_listed_ecs(self): os.close(fd) # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR ecstmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(ecstmpdir, 'easybuild'), parents=True) copy_dir(test_ecs_path, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) @@ -2407,7 +2359,7 @@ def test_header_footer(self): write_file(modules_header, modules_header_txt) # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC # check log message with --skip for existing module args = [ @@ -2441,7 +2393,7 @@ def test_recursive_module_unload(self): """Test generating recursively unloading modules.""" # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + eb_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] @@ -2476,7 +2428,7 @@ def test_tmpdir(self): tmpdir = tempfile.mkdtemp() # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC # check log message with --skip for existing module args = [ @@ -2647,8 +2599,7 @@ def test_allow_modules_tool_mismatch(self): # trigger that main() creates new instance of ModulesTool self.modtool = None - topdir = os.path.abspath(os.path.dirname(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC # keep track of original module definition so we can restore it orig_module = os.environ.get('module', None) @@ -2702,15 +2653,13 @@ def test_allow_modules_tool_mismatch(self): def test_try(self): """Test whether --try options are taken into account.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) - args = [ tweaked_toy_ec, '--dry-run', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, ] test_cases = [ @@ -2764,7 +2713,7 @@ def test_try(self): # Try changing only name or version of toolchain args.pop(0) # Remove EC filename foss_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-foss-2018a.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) + copy_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) write_file(foss_toy_ec, "toolchain['name'] = 'foss'", append=True) test_cases = [ @@ -2779,15 +2728,14 @@ def test_try(self): def test_try_with_copy(self): """Test whether --try options are taken into account.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, '--dry-run', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--copy-ec', ] copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') @@ -2811,9 +2759,7 @@ def test_try_with_copy(self): def test_software_version_ordering(self): """Test whether software versions are correctly ordered when using --software.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - gcc_ec = os.path.join(ecs_path, 'g', 'GCC', 'GCC-4.9.2.eb') + gcc_ec = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.2.eb') test_gcc_ec = os.path.join(self.test_prefix, 'GCC-4.10.1.eb') test_gcc_txt = read_file(gcc_ec).replace("version = '4.9.2'", "version = '4.10.1'") @@ -2823,7 +2769,7 @@ def test_software_version_ordering(self): args = [ '--software=GCC,4.10.1', '--dry-run', - '--robot=%s:%s' % (ecs_path, self.test_prefix), + '--robot=%s:%s' % (TEST_ECS_DIR, self.test_prefix), ] with self.mocked_stdout_stderr(): out = self.eb_main(['--software=GCC,4.10.1'] + args[1:], raise_error=True) @@ -2833,17 +2779,16 @@ def test_software_version_ordering(self): def test_recursive_try(self): """Test whether recursive --try-X works.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "dependencies = [('gzip', '1.4')]\n", append=True) # add fictious dependency - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') + sourcepath = os.path.join(TEST_DIR, 'sandbox', 'sources') args = [ tweaked_toy_ec, '--sourcepath=%s' % sourcepath, '--try-toolchain=gompi,2018a', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--ignore-osdeps', '--dry-run', ] @@ -2906,11 +2851,10 @@ def test_recursive_try(self): def test_cleanup_builddir(self): """Test cleaning up of build dir and --disable-cleanup-builddir.""" - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toy_buildpath = os.path.join(self.test_buildpath, 'toy', '0.0', 'system-system') args = [ - toy_ec, + TOY_EC, '--force', ] with self.mocked_stdout_stderr(): @@ -2927,7 +2871,7 @@ def test_cleanup_builddir(self): # make sure build dir stays in case of failed build args = [ - toy_ec, + TOY_EC, '--force', '--try-amend=prebuildopts=nosuchcommand &&', ] @@ -2937,12 +2881,11 @@ def test_cleanup_builddir(self): def test_filter_deps(self): """Test use of --filter-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + os.environ['MODULEPATH'] = str(TEST_MODULES_DIR) args = [ ec_file, - '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--robot=%s' % TEST_ECS_DIR, '--dry-run', ] with self.mocked_stdout_stderr(): @@ -3051,7 +2994,7 @@ def test_filter_deps(self): # This easyconfig contains a dependency of CMake for which no easyconfig exists. It should still # succeed when called with --filter-deps=CMake=:2.8.10] write_file(self.logfile, '') - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' with self.mocked_stdout_stderr(): @@ -3062,7 +3005,7 @@ def test_filter_deps(self): # The test below fails without PR 2983 write_file(self.logfile, '') - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' with self.mocked_stdout_stderr(): @@ -3071,9 +3014,9 @@ def test_filter_deps(self): def test_hide_deps(self): """Test use of --hide-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + test_dir = TEST_DIR + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + os.environ['MODULEPATH'] = str(TEST_MODULES_DIR) args = [ ec_file, '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), @@ -3108,8 +3051,7 @@ def test_hide_deps(self): def test_hide_toolchains(self): """Test use of --hide-toolchains.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') args = [ ec_file, '--dry-run', @@ -3192,8 +3134,7 @@ def test_parse_http_header_fields_urlpat(self): def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" tmpdir = tempfile.mkdtemp() - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') gzip_ec_txt = read_file(gzip_ec) regex = re.compile('^source_urls = .*', re.M) test_ec_txt = regex.sub("source_urls = ['https://sources.easybuild.io/g/gzip']", gzip_ec_txt) @@ -3311,7 +3252,7 @@ def test_test_report_env_filter(self): def toy(extra_args=None): """Build & install toy, return contents of test report.""" - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC args = [ eb_file, '--force', @@ -3363,7 +3304,7 @@ def test_robot(self): # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp os.environ['EASYBUILD_ROBOT_PATHS'] = self.test_prefix - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR # includes 'toy/.0.0-deps' as a dependency eb_file = os.path.join(test_ecs_path, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') @@ -3689,7 +3630,7 @@ def test_xxx_include_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - test_easyblocks = os.path.dirname(os.path.abspath(__file__)) + test_easyblocks = TEST_DIR path_pattern = os.path.join(test_easyblocks, 'sandbox', 'easybuild', 'easyblocks', 'f', 'foo.py') foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertRegex(logtxt, foo_regex) @@ -3704,7 +3645,7 @@ def test_xxx_include_easyblocks(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -3826,7 +3767,7 @@ def test_xxx_include_generic_easyblocks(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -3970,7 +3911,7 @@ def test_github_xxx_include_easyblocks_from_pr(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic, # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -4039,19 +3980,17 @@ def test_include_module_naming_schemes(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4101,8 +4040,7 @@ def test_use_included_module_naming_scheme(self): ]) write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt) - topdir = os.path.abspath(os.path.dirname(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC args = [ '--unittest-file=%s' % self.logfile, '--module-naming-scheme=AnotherTestIncludedMNS', @@ -4129,19 +4067,17 @@ def test_include_toolchains(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4189,19 +4125,17 @@ def test_include_job_backends(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4237,9 +4171,9 @@ def test_include_job_backends(self): def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" - topdir = os.path.dirname(os.path.abspath(__file__)) + args = [ - os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), + TOY_EC, '--dry-run', '--try-software-version=1.0', # so we get a tweaked easyconfig ] @@ -4272,7 +4206,7 @@ def test_github_preview_pr(self): print("Skipping test_preview_pr, no GitHub token available?") return - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') args = [ '--color=never', @@ -4433,7 +4367,7 @@ def test_minimal_toolchains(self): def test_extended_dry_run(self): """Test use of --extended-dry-run/-x.""" - ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC args = [ ec_file, '--debug', @@ -4502,8 +4436,8 @@ def test_last_log(self): def test_fixed_installdir_naming_scheme(self): """Test use of --fixed-installdir-naming-scheme.""" # by default, name of install dir match module naming scheme used - topdir = os.path.abspath(os.path.dirname(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + eb_file = TOY_EC app = EasyBlock(EasyConfig(eb_file)) app.gen_installdir() self.assertTrue(app.installdir.endswith('software/toy/0.0')) @@ -4543,16 +4477,10 @@ def test_new_branch_github(self): print("Skipping test_create_branch_github, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - - # test easyconfigs - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - args = [ '--new-branch-github', '--github-user=%s' % GITHUB_TEST_ACCOUNT, - toy_ec, + TOY_EC, '-D', ] txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) @@ -4566,7 +4494,7 @@ def test_new_branch_github(self): self.assert_multi_regex(regexs, txt) # test easyblocks - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') toy_eb = os.path.join(test_ebs, 't', 'toy.py') args = [ @@ -4587,7 +4515,7 @@ def test_new_branch_github(self): self.assert_multi_regex(regexs, txt) # test framework with tweaked copy of test_module_naming_scheme.py - test_mns_py = os.path.join(topdir, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', + test_mns_py = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', 'test_module_naming_scheme.py') target_dir = os.path.join(self.test_prefix, 'easybuild-framework', 'test', 'framework', 'sandbox', 'easybuild', 'tools', 'module_naming_scheme') @@ -4659,14 +4587,10 @@ def test_update_branch_github(self): print("Skipping test_update_branch_github, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - args = [ '--update-branch-github=develop', '--github-user=boegel', # used to determine account to grab branch from (no GitHub token needed) - toy_ec, + TOY_EC, '--pr-commit-msg="this is just a test"', '--force', # force required because we're using --pr-commit-msg when only adding new easyconfigs '-D', @@ -4690,11 +4614,11 @@ def test_github_new_update_pr(self): return # copy toy test easyconfig - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') + + test_ecs = TEST_ECS_DIR toy_ec = os.path.join(self.test_prefix, 'toy.eb') toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(topdir, 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) # purposely picked one with non-default toolchain/versionsuffix copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), toy_ec) @@ -4917,10 +4841,8 @@ def test_github_new_pr_warning_missing_patch(self): print("Skipping test_new_pr_warning_missing_patch, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), test_ec) + copy_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), test_ec) patches_regex = re.compile(r'^patches = .*', re.M) test_ec_txt = read_file(test_ec) @@ -5017,9 +4939,9 @@ def test_github_new_pr_python(self): return # copy toy test easyconfig - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR toy_ec = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), toy_ec) + copy_file(TOY_EC, toy_ec) # modify file to include Python dependency toy_ec_txt = read_file(toy_ec) @@ -5138,8 +5060,7 @@ def test_github_new_pr_easyblock(self): print("Skipping test_new_pr_easyblock, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertExists(toy_eb) args = [ @@ -5321,7 +5242,7 @@ def test_show_config(self): default_prefix = os.path.join(os.environ['HOME'], '.local', 'easybuild') - test_dir = os.path.dirname(os.path.abspath(__file__)) + test_dir = TEST_DIR expected_lines = [ r"#", r"# Current EasyBuild configuration", @@ -5334,7 +5255,7 @@ def test_show_config(self): r"ignoreconfigfiles\s* \(E\) = %s" % ', '.join(os.environ['EASYBUILD_IGNORECONFIGFILES'].split(',')), r"installpath\s* \(E\) = " + os.path.join(self.test_prefix, 'tmp.*'), r"repositorypath\s* \(D\) = " + os.path.join(default_prefix, 'ebfiles_repo'), - r"robot-paths\s* \(E\) = " + os.path.join(test_dir, 'easyconfigs', 'test_ecs'), + r"robot-paths\s* \(E\) = " + os.path.join(TEST_ECS_DIR), r"rpath\s* \(D\) = " + ('False' if get_os_type() == DARWIN else 'True'), r"sourcepath\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'sources'), r"sourcepath-data\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'data_sources'), @@ -5617,8 +5538,7 @@ def test_fetch_all(self): mkdir(lock_path, parents=True) # copy toy-0.0.eb test easyconfig, tweak version to something that no source can be obtained for - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT test_ec_txt = toy_ec_txt.replace("version = '0.0'", "version = '1.2.3.4.5.6'") test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -5842,12 +5762,11 @@ def test_list_software(self): # copy selected test easyconfigs for testing --list-*software options with; # full test is a nuisance, because all dependencies must be available and toolchains like intel must have # all expected components when testing with HierarchicalMNS (which the test easyconfigs don't always have) - topdir = os.path.dirname(os.path.abspath(__file__)) - cray_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb') - gcc_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb') - gzip_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') - gzip_system_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb') + cray_ec = os.path.join(TEST_ECS_DIR, 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb') + gcc_ec = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') + gzip_ec = os.path.join(TEST_DIR, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + gzip_system_ec = os.path.join(TEST_DIR, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb') test_ecs = os.path.join(self.test_prefix, 'test_ecs') for ec in [cray_ec, gcc_ec, gzip_ec, gzip_system_ec]: @@ -6023,7 +5942,7 @@ def test_check_contrib_style(self): # copy toy-0.0.eb test easyconfig, fiddle with it to make style check fail toy = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), toy) + copy_file(TOY_EC, toy) toytxt = read_file(toy) # introduce whitespace issues @@ -6078,7 +5997,7 @@ def test_check_contrib_non_style(self): # --check-contrib passes if None values are used as checksum, but produces warning toy = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), toy) + copy_file(TOY_EC, toy) toytxt = read_file(toy) toytxt = toytxt + '\n'.join([ 'checksums = [', @@ -6118,13 +6037,11 @@ def test_allow_use_as_root(self): def test_verify_easyconfig_filenames(self): """Test --verify-easyconfig-filename""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - toy_ec = os.path.join(test_easyconfigs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) args = [ test_ec, @@ -6151,7 +6068,7 @@ def test_verify_easyconfig_filenames(self): write_file(self.logfile, '') - args[0] = toy_ec + args[0] = TOY_EC with self.mocked_stdout_stderr(): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) @@ -6159,8 +6076,8 @@ def test_verify_easyconfig_filenames(self): def test_set_default_module(self): """Test use of --set-default-module""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') with self.mocked_stdout_stderr(): self.eb_main([toy_ec, '--set-default-module'], do_build=True, raise_error=True) @@ -6264,8 +6181,8 @@ def test_set_default_module_robot(self): def test_inject_checksums(self): """Test for --inject-checksums""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # checksums are injected in existing easyconfig, so test with a copy test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -6401,8 +6318,7 @@ def test_inject_checksums(self): remove_file(ec_backups[0]) # also test injecting of MD5 checksums into easyconfig that doesn't include checksums already - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT # get rid of existing checksums regex = re.compile(r'^checksums(?:.|\n)*?\]\s*$', re.M) @@ -6444,7 +6360,7 @@ def test_inject_checksums(self): self.assertEqual(ec['checksums'], checksums) # check whether empty list of checksums is stripped out by --inject-checksums - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT regex = re.compile(r'^checksums(?:.|\n)*?\]\s*$', re.M) toy_ec_txt = regex.sub('', toy_ec_txt) @@ -6505,7 +6421,7 @@ def test_inject_checksums(self): self.assertEqual(ext_opts['checksums'], expected_checksums) # Also works for cargo crates - cargo_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-cargo.eb') + cargo_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-cargo.eb') copy_file(cargo_ec, test_ec) stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True) self.assertIn("injecting sha256 checksums in", stdout) @@ -6567,10 +6483,8 @@ def test_inject_checksums(self): def test_inject_checksums_to_json(self): """Test --inject-checksums-to-json.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) test_ec_txt = read_file(test_ec) args = [test_ec, '--inject-checksums-to-json'] @@ -6596,15 +6510,14 @@ def test_inject_checksums_to_json(self): def test_force_download(self): """Test --force-download""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_srcdir = os.path.join(topdir, 'sandbox', 'sources', 'toy') - copy_file(toy_ec, self.test_prefix) + toy_srcdir = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') + + copy_file(TOY_EC, self.test_prefix) toy_tar = 'toy-0.0.tar.gz' copy_file(os.path.join(toy_srcdir, toy_tar), os.path.join(self.test_prefix, 't', 'toy', toy_tar)) - toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) + toy_ec = os.path.join(self.test_prefix, os.path.basename(TOY_EC)) write_file(toy_ec, "\nsource_urls = ['file://%s']" % toy_srcdir, append=True) args = [ @@ -6624,8 +6537,8 @@ def test_force_download(self): def test_enforce_checksums(self): """Test effect of --enforce-checksums""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') # wipe $EASYBUILD_ROBOT_PATHS to avoid that checksums.json for toy is found in test_ecs @@ -6724,9 +6637,6 @@ def test_check_eb_deps(self): def test_tmp_logdir(self): """Test use of --tmp-logdir.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - # purposely use a non-existing directory as log directory tmp_logdir = os.path.join(self.test_prefix, 'tmp-logs') self.assertNotExists(tmp_logdir) @@ -6736,7 +6646,7 @@ def test_tmp_logdir(self): # check log message with --skip for existing module args = [ - toy_ec, + TOY_EC, '--force', '--debug', '--tmp-logdir=%s' % tmp_logdir, @@ -6752,11 +6662,9 @@ def test_tmp_logdir(self): def test_sanity_check_only(self): """Test use of --sanity-check-only.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.ec') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", @@ -6865,7 +6773,7 @@ def test_sanity_check_only(self): self.eb_main(args, do_build=True, raise_error=True) # also check when using easyblock that enables build_in_installdir in its constructor - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') toy_eb = os.path.join(test_ebs, 't', 'toy.py') toy_eb_txt = read_file(toy_eb) @@ -6899,11 +6807,9 @@ def test_sanity_check_only(self): def test_keep_going(self): """Test use of --keep-going.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nsources=["toy-0.0.tar.gz"]' write_file(test_ec, test_ec_txt + '\nversion="broken"\npreconfigopts = "false && "') test_ec2 = os.path.join(self.test_prefix, 'test2.eb') @@ -6936,12 +6842,10 @@ def test_keep_going(self): def test_skip_extensions(self): """Test use of --skip-extensions.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # add extension, which should be skipped test_ec = os.path.join(self.test_prefix, 'test.ec') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "exts_list = [", " ('barbar', '0.0', {", @@ -6970,9 +6874,6 @@ def test_skip_extensions(self): def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_mns = os.path.join(self.test_prefix, 'test_mns.py') test_mns_txt = '\n'.join([ "import vsc", @@ -6983,7 +6884,7 @@ def test_fake_vsc_include(self): write_file(test_mns, test_mns_txt) args = [ - toy_ec, + TOY_EC, '--dry-run', '--include-module-naming-schemes=%s' % test_mns, ] @@ -6996,22 +6897,19 @@ def test_fake_vsc_include(self): def test_installdir(self): """Check naming scheme of installation directory.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/toy/0.0')) # even with HierarchicalMNS the installation directory remains the same, # due to --fixed-installdir-naming-scheme being enabled by default args = ['--module-naming-scheme=HierarchicalMNS'] init_config(args=args) - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/toy/0.0')) # things change when --disable-fixed-installdir-naming-scheme is used init_config(args=args, build_options={'fixed_installdir_naming_scheme': False}) - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/Core/toy/0.0')) def test_cuda_compute_capabilities(self): @@ -7024,7 +6922,7 @@ def test_cuda_compute_capabilities(self): def test_create_index(self): """Test --create-index option.""" - test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR remove_dir(self.test_prefix) copy_dir(test_ecs, self.test_prefix) @@ -7113,13 +7011,12 @@ def test_accept_eula_for(self): """Test --accept-eula-for configuration option.""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ "easyblock = 'EB_toy_eula'", '', - read_file(toy_ec), + TOY_EC_TXT, ]) write_file(test_ec, test_ec_txt) @@ -7239,8 +7136,8 @@ def test_config_repositorypath(self): # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') + + toy_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_nonexistent.yaml') args = ['--easystack', toy_easystack, '--experimental'] expected_err = "No such file or directory: '%s'" % toy_easystack with self.mocked_stdout_stderr(): @@ -7250,8 +7147,8 @@ def test_easystack_wrong_read(self): # expecting successful build def test_easystack_basic(self): """Test for --easystack -> success case""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + toy_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_basic.yaml') args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run'] with self.mocked_stdout_stderr(): @@ -7334,12 +7231,11 @@ def test_easystack_easyconfigs_cache(self): Test for easystack file that specifies same easyconfig twice, but from a different location. """ - topdir = os.path.abspath(os.path.dirname(__file__)) - libtoy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'l', 'libtoy', 'libtoy-0.0.eb') - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + libtoy_ec = os.path.join(TEST_ECS_DIR, 'l', 'libtoy', 'libtoy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\ndependencies = [('libtoy', '0.0')]" write_file(test_ec, test_ec_txt) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index b698ea013a..dde290348e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,6 +38,7 @@ import unittest from contextlib import contextmanager from importlib import reload +from pathlib import Path from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -316,7 +317,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal # always run main in unit testing mode (which for example allows for using deprecated toolchains); # note: don't change 'args' value, which is passed by reference! - main_args = args + ['--unit-testing-mode'] + main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode'] myerr = False if logfile is None: