From 365a5e1cb296c41883659ffc2fef40478c97d01a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Jun 2026 14:12:44 +0200 Subject: [PATCH] Use constants for commonly used test easyconfig folders, files and content --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/tools/filetools.py | 3 +- test/framework/__init__.py | 10 + test/framework/config.py | 7 +- test/framework/containers.py | 23 +- test/framework/docs.py | 8 +- test/framework/easyblock.py | 181 ++++----- test/framework/easyconfig.py | 272 +++++--------- test/framework/easyconfigparser.py | 21 +- test/framework/easystack.py | 23 +- test/framework/filetools.py | 155 ++++---- test/framework/github.py | 15 +- test/framework/include.py | 8 +- test/framework/lib.py | 3 +- test/framework/module_generator.py | 33 +- test/framework/modules.py | 37 +- test/framework/modulestool.py | 3 +- test/framework/options.py | 376 +++++++------------ test/framework/package.py | 11 +- test/framework/parallelbuild.py | 42 +-- test/framework/repository.py | 11 +- test/framework/robot.py | 154 ++++---- test/framework/style.py | 5 +- test/framework/toolchain.py | 21 +- test/framework/toy_build.py | 27 +- test/framework/tweak.py | 94 ++--- test/framework/utilities.py | 21 +- 27 files changed, 607 insertions(+), 959 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 698891a772..5255a5b614 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2327,7 +2327,7 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, # only cache when no build specifications are involved (since those can't be part of a dict key) cache_key = None if not build_specs: - cache_key = (path, validate, hidden, parse_only) + cache_key = (str(path), validate, hidden, parse_only) if cache_key in _easyconfigs_cache: # Note: This does NOT copy EasyConfig instances but the dict containing an instance in the 'ec' key. # So modifications to the `EasyConfig` instance will be shared. diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..120eb9640a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, str): + if isinstance(paths, (str, Path)): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/test/framework/__init__.py b/test/framework/__init__.py index 895b15e622..04daadf7fa 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -27,3 +27,13 @@ @author: Toon Willems (Ghent University) """ + +from pathlib import Path +from easybuild.tools.filetools import read_file + +TEST_DIR = Path(__file__).parent +REPO_ROOT = TEST_DIR.parent.parent +TEST_MODULES_DIR = TEST_DIR / 'modules' +TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs' +TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb' +TOY_EC_TXT: str = read_file(TOY_EC) diff --git a/test/framework/config.py b/test/framework/config.py index 03d7c3fbf3..3ae1c972bf 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -34,6 +34,7 @@ import sys import tempfile from importlib import reload +from test.framework import TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -264,8 +265,7 @@ def test_generaloption_config_file(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + copy_dir(os.path.join(TEST_ECS_DIR), os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -498,9 +498,8 @@ def test_flex_robot_paths(self): # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') tmp_ecs_dir = os.path.join(tmpdir, 'easybuild', 'easyconfigs') - copy_dir(test_ecs_path, tmp_ecs_dir) + copy_dir(os.path.join(TEST_ECS_DIR), tmp_ecs_dir) # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default orig_sys_path = sys.path[:] diff --git a/test/framework/containers.py b/test/framework/containers.py index 93fc04548c..1c47ccfc45 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -31,6 +31,7 @@ import re import stat import sys +from test.framework import TEST_ECS_DIR, TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -89,9 +90,6 @@ def check_regexs(self, regexs, stdout): def test_end2end_singularity_recipe_config(self): """End-to-end test for --containerize (recipe only), using --container-config.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) @@ -100,7 +98,7 @@ def test_end2end_singularity_recipe_config(self): test_container_recipe = os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0') args = [ - toy_ec, + TOY_EC, '--containerize', '--experimental', ] @@ -257,9 +255,6 @@ def test_end2end_singularity_recipe_config(self): def test_end2end_singularity_image(self): """End-to-end test for --containerize (recipe + image).""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) @@ -269,7 +264,7 @@ def test_end2end_singularity_image(self): write_file(test_img, '') args = [ - toy_ec, + TOY_EC, '-C', # equivalent with --containerize '--experimental', '--container-config=bootstrap=localimage,from=%s' % test_img, @@ -360,16 +355,13 @@ def test_end2end_singularity_image(self): self.check_regexs(regexs, stdout) def test_end2end_dockerfile(self): - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) base_args = [ - toy_ec, + TOY_EC, '--containerize', '--container-type=docker', '--experimental', @@ -402,7 +394,7 @@ def test_end2end_dockerfile(self): remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) - base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb')) + base_args.insert(1, os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.2.eb')) self.run_main(base_args + ['--container-config=ubuntu:20.04']) def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) regexs = [ @@ -419,16 +411,13 @@ def test_end2end_dockerfile(self): def test_end2end_docker_image(self): - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) args = [ - toy_ec, + TOY_EC, '-C', # equivalent with --containerize '--experimental', '--container-type=docker', diff --git a/test/framework/docs.py b/test/framework/docs.py index b24138074e..72af821eda 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -39,6 +39,7 @@ from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_DIR, TEST_ECS_DIR LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock @@ -869,8 +870,7 @@ def test_list_easyblocks(self): """ Tests for list_easyblocks function """ - topdir = os.path.dirname(os.path.abspath(__file__)) - topdir_easyblocks = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + topdir_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') txt = list_easyblocks() self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_TXT) @@ -899,7 +899,7 @@ def test_list_easyblocks(self): def test_list_software(self): """Test list_software* functions.""" build_options = { - 'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')], + 'robot_path': [os.path.join(TEST_DIR, 'easyconfigs', 'v1.0')], 'silent': True, 'valid_module_classes': module_classes(), } @@ -933,7 +933,7 @@ def test_list_software(self): # check for specific patterns in output for larger set of test easyconfigs build_options = { - 'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 25760f1db8..3448ff6ffa 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -44,6 +44,7 @@ from inspect import cleandoc from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from unittest import TextTestRunner import easybuild.tools.systemtools as st @@ -172,8 +173,7 @@ def test_load_module(self): tmp_modules = os.path.join(self.test_prefix, 'modules') mkdir(tmp_modules) - test_dir = os.path.abspath(os.path.dirname(__file__)) - copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) + copy_dir(os.path.join(TEST_MODULES_DIR, 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '2.1.2-GCC-6.4.0-2.28') ompi_mod_txt = read_file(openmpi_module) @@ -223,12 +223,10 @@ def test_load_module(self): # test HMNS module load when conflicting dependencies are available in both Core and # toolchain-specific modulepaths # see also https://github.com/easybuilders/easybuild-framework/issues/4986 - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'easyconfigs', 'test_ecs') os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' build_options = { 'generate_devel_module': True, # go through EasyBlock.fake_module_environment() - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], } init_config(build_options=build_options) @@ -236,9 +234,8 @@ def test_load_module(self): mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'modules', 'HierarchicalMNS', mod_subdir) - copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir)) + copy_dir(os.path.join(TEST_MODULES_DIR, 'HierarchicalMNS', mod_subdir), + os.path.join(mod_prefix, mod_subdir)) # tweak use statements in toolchain module to ensure correct paths modfile = os.path.join(mod_prefix, 'Core', 'GCCcore', '12.3.0') @@ -248,7 +245,7 @@ def test_load_module(self): line) sys.stdout.write(line) - test_eb_file = os.path.join(test_ecs_path, 'g', 'GLib', 'GLib-2.77.1-GCCcore-12.3.0.eb') + test_eb_file = os.path.join(TEST_ECS_DIR, 'g', 'GLib', 'GLib-2.77.1-GCCcore-12.3.0.eb') eb = EasyBlock(EasyConfig(test_eb_file)) self.reset_modulepath([os.path.join(mod_prefix)]) @@ -1078,11 +1075,10 @@ def test_make_module_dep(self): def test_make_module_dep_hmns(self): """Test for make_module_dep under HMNS""" - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_stops': all_stops, 'validate': False, @@ -1123,11 +1119,10 @@ def test_make_module_dep_hmns(self): def test_make_module_dep_of_dep_hmns(self): """Test for make_module_dep under HMNS with dependencies of dependencies""" - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1203,11 +1198,8 @@ def test_det_iter_cnt(self): def test_handle_iterate_opts(self): """Test for handle_iterate_opts method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec) + "\nconfigopts = ['--opt1 --anotheropt', '--opt2', '--opt3 --optbis']") + write_file(test_ec, TOY_EC_TXT + "\nconfigopts = ['--opt1 --anotheropt', '--opt2', '--opt3 --optbis']") ec = process_easyconfig(test_ec)[0] eb = get_easyblock_instance(ec) @@ -1383,15 +1375,12 @@ def test_post_processing_step(self): """Test post_processing_step and deprecated post_install_step.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - # these imports only work here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy from easybuild.easyblocks.toy_deprecated import EB_toy_deprecated cwd = os.getcwd() - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy_deprecated(toy_ec) eb.silent = True depr_msg = r"EasyBlock.post_install_step\(\) is deprecated, use EasyBlock.post_processing_step\(\) instead" @@ -1400,7 +1389,7 @@ def test_post_processing_step(self): self.assertErrorRegex(EasyBuildError, expected_error, eb.run_all_steps, True) change_dir(cwd) - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy(toy_ec) eb.silent = True with self.mocked_stdout_stderr() as (_, stderr), self.saved_env(): @@ -1418,7 +1407,7 @@ def test_post_processing_step(self): change_dir(cwd) self.allow_deprecated_behaviour() - toy_ec = EasyConfig(toy_ec_fn) + toy_ec = EasyConfig(TOY_EC) eb = EB_toy_deprecated(toy_ec) eb.silent = True with self.mocked_stdout_stderr() as (stdout, stderr), self.saved_env(): @@ -1527,8 +1516,7 @@ def test_extensions_step_deprecations(self): def test_init_extensions(self): """Test creating extension instances.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_txt = read_file(toy_ec_file) test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1992,9 +1980,8 @@ def test_make_builddir(self): def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" from easybuild.easyblocks.toy import EB_toy - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) self.assertIsInstance(eb, EB_toy) @@ -2006,11 +1993,10 @@ def test_get_easyblock_instance(self): def test_fetch_sources(self): """Test fetch_sources method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) - toy_source = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_source = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') with self.mocked_stdout_stderr(): eb.fetch_sources() @@ -2081,8 +2067,7 @@ def test_fetch_sources(self): def test_fetch_sources_git(self): """Test fetch_sources method from git repo.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) eb.src = [] sources = [ @@ -2223,8 +2208,7 @@ def test_download_instructions(self): def test_fetch_patches(self): """Test fetch_patches method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) toy_patch = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' @@ -2257,8 +2241,7 @@ def test_fetch_patches(self): self.assertEqual(eb.patches[3]['copy'], 'some/path') self.assertEqual(eb.patches[4]['name'], toy_patch) self.assertEqual(eb.patches[4]['level'], 0) - testdir = os.path.abspath(os.path.dirname(__file__)) - sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') + sandbox_sources = os.path.join(TEST_DIR, 'sandbox', 'sources') self.assertEqual(eb.patches[4]['path'], os.path.join(sandbox_sources, 'alt_toy', toy_patch)) patches = [ @@ -2269,8 +2252,7 @@ def test_fetch_patches(self): def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' - testdir = os.path.abspath(os.path.dirname(__file__)) - sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') + sandbox_sources = os.path.join(TEST_DIR, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) alt_toy_tarball_path = os.path.join(sandbox_sources, 'alt_toy', toy_tarball) tmpdir = tempfile.mkdtemp() @@ -2278,15 +2260,14 @@ def test_obtain_file(self): mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(tmpdir, 'ecs', 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) ec = process_easyconfig(test_ec)[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works - init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) + init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, TEST_DIR)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) with self.mocked_stdout_stderr(): res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) @@ -2428,10 +2409,9 @@ def test_fallback_source_url(self): def test_collect_exts_file_info(self): """Test collect_exts_file_info method.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy') + toy_sources = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') toy_ext_sources = os.path.join(toy_sources, 'extensions') - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') new_ext_txt = "('baz', '0.0', {'nosource': True})," # With nosource option @@ -2526,8 +2506,7 @@ def test_collect_exts_file_info(self): def test_obtain_file_extension(self): """Test use of obtain_file method on an extension.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec = process_easyconfig(toy_ec_file)[0] toy_eb = EasyBlock(toy_ec['ec']) @@ -2548,8 +2527,7 @@ def test_check_readiness(self): # check that check_readiness step works (adding dependencies, etc.) ec_file = 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb' - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'o', 'OpenMPI', ec_file) + ec_path = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', ec_file) ec = EasyConfig(ec_path) eb = EasyBlock(ec) eb.check_readiness_step() @@ -2575,11 +2553,10 @@ def test_exclude_path_to_top_of_module_tree(self): w.r.t. not including any load statements for modules that build up the path to the top of the module tree. """ self.orig_module_naming_scheme = config.get_module_naming_scheme() - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_path], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -2603,7 +2580,7 @@ def test_exclude_path_to_top_of_module_tree(self): ('i/imkl/imkl-11.3.1.150-iimpi-2016.01.eb', imkl_modfile_path, iccifort_mods + ['iimpi', 'impi']), ] for ec_file, modfile_path, excluded_deps in tests: - ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, ec_file)) eb = EasyBlock(ec) with self.mocked_stdout_stderr(): eb.toolchain.prepare() @@ -2635,9 +2612,7 @@ def test_patch_step(self): """Test patch step.""" cwd = os.getcwd() - testdir = os.path.abspath(os.path.dirname(__file__)) - test_easyconfigs = os.path.join(testdir, 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]['ec'] + ec = process_easyconfig(TOY_EC)[0]['ec'] orig_sources = ec['sources'][:] toy_patches = [ @@ -2700,8 +2675,7 @@ def test_extensions_sanity_check(self): """Test sanity check aspect of extensions.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_fn = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # Do this before loading the easyblock to check the non-translated output below os.environ['LC_ALL'] = 'C' @@ -2739,29 +2713,25 @@ def test_extensions_sanity_check(self): def test_parallel(self): """Test defining of parallelism.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toytxt = read_file(toy_ec) - handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec1, toytxt + "\nparallel = 13") + write_file(toy_ec1, TOY_EC_TXT + "\nparallel = 13") handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 12\nmaxparallel = 6") + write_file(toy_ec2, TOY_EC_TXT + "\nparallel = 12\nmaxparallel = 6") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec3, toytxt + "\nparallel = False") + write_file(toy_ec3, TOY_EC_TXT + "\nparallel = False") handle, toy_ec4 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec4, toytxt + "\nmaxparallel = 6") + write_file(toy_ec4, TOY_EC_TXT + "\nmaxparallel = 6") handle, toy_ec5 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec5, toytxt + "\nmaxparallel = False") + write_file(toy_ec5, TOY_EC_TXT + "\nmaxparallel = False") # default: parallelism is derived from # available cores + ulimit # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here @@ -2787,7 +2757,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2823,7 +2793,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2860,7 +2830,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2878,7 +2848,7 @@ def test_parallel(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2888,7 +2858,7 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['parallel'], expected) # Template updated correctly - self.contents = toytxt + '\nmaxparallel=2' + self.contents = TOY_EC_TXT + '\nmaxparallel=2' self.writeEC() test_eb = EasyBlock(EasyConfig(self.eb_file)) test_eb.post_init() @@ -2906,7 +2876,7 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['buildopts'], '-j 1') # Legacy behavior. To be removed after deprecation of the parallel EC parameter - self.contents = toytxt + '\nmaxparallel=99' + self.contents = TOY_EC_TXT + '\nmaxparallel=99' self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) @@ -2926,10 +2896,6 @@ def test_parallel(self): def test_keepsymlinks(self): """Test keepsymlinks parameter (default: True).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toytxt = read_file(toy_ec) - test_cases = { '': True, 'keepsymlinks = False': False, @@ -2938,7 +2904,7 @@ def test_keepsymlinks(self): for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): - self.contents = toytxt + '\n' + txt + self.contents = TOY_EC_TXT + '\n' + txt self.writeEC() test_eb = EasyBlock(EasyConfig(self.eb_file)) test_eb.post_init() @@ -2946,8 +2912,7 @@ def test_keepsymlinks(self): def test_guess_start_dir(self): """Test guessing the start dir.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] cwd = os.getcwd() self.assertExists(cwd) @@ -2984,8 +2949,7 @@ def check_start_dir(expected_start_dir): def test_extension_set_start_dir(self): """Test start dir with extensions.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] cwd = os.getcwd() self.assertExists(cwd) @@ -3090,8 +3054,7 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=Non def test_extension_patch_step(self): """Test start dir with extensions.""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]['ec'] + ec = process_easyconfig(TOY_EC)[0]['ec'] cwd = os.getcwd() self.assertExists(cwd) @@ -3152,8 +3115,7 @@ def run_extension_step(): def test_prepare_step(self): """Test prepare step (setting up build environment).""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'system-system'), parents=True) eb = EasyBlock(ec['ec']) @@ -3181,8 +3143,7 @@ def test_prepare_step_load_tc_deps_modules(self): init_config(build_options={'robot_path': os.environ['EASYBUILD_ROBOT_PATHS']}) - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ompi_ec_file = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + ompi_ec_file = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') ec = process_easyconfig(ompi_ec_file, validate=False)[0] mkdir(os.path.join(self.test_buildpath, 'OpenMPI', '2.1.2', 'GCC-6.4.0-2.28'), parents=True) @@ -3215,10 +3176,9 @@ def test_prepare_step_hmns(self): """ Check whether loading of already existing dependencies during prepare step works when HierarchicalMNS is used. """ - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' - init_config(build_options={'robot_path': [test_ecs]}) + init_config(build_options={'robot_path': [TEST_ECS_DIR]}) # set up hierarchical modules, but reset $MODULEPATH to empty # the expectation is that EasyBuild set's up the $MODULEPATH such that pre-installed dependencies can be loaded @@ -3230,11 +3190,9 @@ def test_prepare_step_hmns(self): self.reset_modulepath([]) self.assertEqual(os.environ.get('MODULEPATH'), None) - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') regex = re.compile('^toolchain = .*', re.M) - test_ectxt = regex.sub("toolchain = SYSTEM", read_file(toy_ec)) + test_ectxt = regex.sub("toolchain = SYSTEM", TOY_EC_TXT) test_ectxt += "\ndependencies = [('GCC', '6.4.0', '-2.28')]" write_file(test_ec, test_ectxt) @@ -3254,9 +3212,7 @@ def test_prepare_step_cuda_cache(self): init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode - test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - ec = process_easyconfig(toy_ec)[0] + ec = process_easyconfig(TOY_EC)[0] eb = EasyBlock(ec['ec']) eb.silent = True with self.mocked_stdout_stderr(): @@ -3270,7 +3226,7 @@ def test_prepare_step_cuda_cache(self): # Now with CUDA test_ec = os.path.join(self.test_prefix, 'test.eb') test_ectxt = re.sub('^toolchain = .*', "toolchain = {'name': 'gcccuda', 'version': '2018a'}", - read_file(toy_ec), flags=re.M) + TOY_EC_TXT, flags=re.M) write_file(test_ec, test_ectxt) ec = process_easyconfig(test_ec)[0] eb = EasyBlock(ec['ec']) @@ -3310,8 +3266,7 @@ def test_prepare_step_cuda_cache(self): def test_checksum_step(self): """Test checksum step""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = process_easyconfig(toy_ec)[0] eb = get_easyblock_instance(ec) @@ -3362,7 +3317,7 @@ def test_checksum_step(self): # create test easyconfig from which checksums have been stripped test_ec = os.path.join(self.test_prefix, 'test.eb') - ectxt = read_file(toy_ec) + ectxt = TOY_EC_TXT regex = re.compile(r"'?checksums'?\s*[=:]\s*\[[^]]+\].*", re.M) ectxt = regex.sub('', ectxt) write_file(test_ec, ectxt) @@ -3380,7 +3335,7 @@ def test_checksum_step(self): self.fail("Incorrect extension type: %s" % type(ext)) # put checksums.json in place next to easyconfig file being used for the tests - toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums.json') + toy_checksums_json = os.path.join(TEST_ECS_DIR, 't', 'toy', 'checksums.json') copy_file(toy_checksums_json, os.path.join(self.test_prefix, 'checksums.json')) # test without checksums, it should work since they are in checksums.json @@ -3436,8 +3391,7 @@ def test_checksum_step(self): def test_check_checksums(self): """Test for check_checksums_for and check_checksums methods.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = process_easyconfig(toy_ec)[0] eb = get_easyblock_instance(ec) @@ -3465,9 +3419,7 @@ def run_checks(): self.assertIn(ext_error_tmpl % ext, line) # check whether tuple of alternative SHA256 checksums is correctly recognized - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - ec = process_easyconfig(toy_ec)[0] + ec = process_easyconfig(TOY_EC)[0] eb = get_easyblock_instance(ec) # single SHA256 checksum per source/patch: OK @@ -3545,7 +3497,7 @@ def run_checks(): # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) ec = process_easyconfig(test_ec)[0] eb = get_easyblock_instance(ec) eb.cfg['checksums'] = [] @@ -3556,7 +3508,7 @@ def run_checks(): self.assertEqual(res[0], expected) # all is fine if checksums.json is also copied - copy_file(os.path.join(os.path.dirname(toy_ec), 'checksums.json'), self.test_prefix) + copy_file(os.path.join(os.path.dirname(TOY_EC), 'checksums.json'), self.test_prefix) eb.json_checksums = None self.assertEqual(eb.check_checksums(), []) @@ -3763,9 +3715,7 @@ def test_arch_specific_sanity_check(self): def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) eb.dry_run = True error_pattern = r"Incorrect format for sanity_check_paths: " @@ -3870,9 +3820,6 @@ def test_report_current_step_method(self): """ Check whether name of methods in installation steps are correctly reported """ - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - class MockEasyBlock(EasyBlock): # Mock methods def build_step(self): @@ -3885,7 +3832,7 @@ def test_step(self): def custom_step(self): self.log.info('Ran custom') - eb = MockEasyBlock(EasyConfig(toy_ec)) + eb = MockEasyBlock(EasyConfig(TOY_EC)) # Part of run_all_steps steps = [step for step in eb.get_steps() if step[0] == BUILD_STEP] for step_name, _, step_methods, _ in steps: @@ -3911,10 +3858,8 @@ def test_exts_deps_build_env(self): Test whether dependencies are loaded in build environment for extensions. """ # to verify fix made in https://github.com/easybuilders/easybuild-framework/pull/5023 - testdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += textwrap.dedent(""" toolchain = {'name': 'gompi', 'version': '2023a'} @@ -3946,7 +3891,7 @@ def test_exts_deps_build_env(self): test_mods = os.path.join(self.test_prefix, 'modules') for name, mod_fn in mod_files: - mod_fp = os.path.join(testdir, 'modules', name, mod_fn) + mod_fp = os.path.join(TEST_MODULES_DIR, name, mod_fn) header_fn = 'zlib.h' if name == 'zlib' else 'mpi.h' @@ -4006,7 +3951,7 @@ def test_exts_deps_build_env(self): self.assertRegex(log_txt, regex) # verify fix made in https://github.com/easybuilders/easybuild-framework/pull/5048 - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += textwrap.dedent(""" toolchain = {'name': 'GCCcore', 'version': '12.3.0'} """) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96b7f47307..76fcf96f7d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -40,6 +40,7 @@ import textwrap from collections import OrderedDict from easybuild.tools import LooseVersion +from test.framework import TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -460,10 +461,9 @@ def test_extra_options(self): def test_exts_list(self): """Test handling of list of extensions.""" - topdir = os.path.dirname(os.path.abspath(__file__)) os.environ['EASYBUILD_SOURCEPATH'] = ':'.join([ - os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip'), - os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy'), + os.path.join(TEST_ECS_DIR, 'g', 'gzip'), + os.path.join(TEST_ECS_DIR, 't', 'toy'), ]) init_config() self.contents = textwrap.dedent(""" @@ -850,11 +850,9 @@ def test_alt_easyconfig_paths(self): def test_tweak_multiple_tcs(self): """Test that tweaking variables of ECs from multiple toolchains works""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # Create directories to store the tweaked easyconfigs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + robot_path = det_robot_path([TEST_ECS_DIR], tweaked_ecs_paths, pr_path, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), @@ -863,8 +861,8 @@ def test_tweak_multiple_tcs(self): }) # Allow tweaking of non-toolchain values for multiple ECs of different toolchains - untweaked_openmpi_1 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') - untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') + untweaked_openmpi_1 = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi_2 = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)]) tweak_specs = {'moduleclass': 'debugger'} easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths, @@ -1327,8 +1325,7 @@ def test_templating_constants(self): self.assertEqual(ec['source_urls'][3], 'https://github.com/pi/pi/releases/download/v3.04') # test use of %(mpi_cmd_prefix)s template - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - gompi_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a.eb') + gompi_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, read_file(gompi_ec) + "\nsanity_check_commands = ['%(mpi_cmd_prefix)s toy']") @@ -1509,8 +1506,7 @@ def test_templating_doc(self): def test_start_dir_template(self): """Test the %(startdir)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + ec = process_easyconfig(TOY_EC)[0] self.contents = textwrap.dedent(""" name = 'toy' @@ -1554,11 +1550,9 @@ def test_start_dir_template(self): def test_rpath_template(self): """Test the %(rpath)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "configopts = '--with-rpath=%(rpath_enabled)s'" write_file(test_ec, test_ec_txt) @@ -1579,11 +1573,8 @@ def test_rpath_template(self): def test_sysroot_template(self): """Test the %(sysroot)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"' test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"' @@ -1607,11 +1598,8 @@ def test_sysroot_template(self): def test_software_commit_template(self): """Test the %(software_commit)s template""" - test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"' test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"' test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"' @@ -1794,7 +1782,7 @@ def test_format_equivalence_basic(self): orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True - easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs') + easyconfigs_path = TEST_DIR / 'easyconfigs' # set max diff high enough to make sure the difference is shown in case of problems self.maxDiff = 10000 @@ -1825,19 +1813,16 @@ def test_format_equivalence_basic(self): def test_fetch_parameters_from_easyconfig(self): """Test fetch_parameters_from_easyconfig function.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_file = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - for ec_file, correct_name, correct_easyblock in [ - (toy_ec_file, 'toy', None), - (os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb'), 'foss', 'Toolchain'), + (TOY_EC, 'toy', None), + (os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb'), 'foss', 'Toolchain'), ]: name, easyblock = fetch_parameters_from_easyconfig(read_file(ec_file), ['name', 'easyblock']) self.assertEqual(name, correct_name) self.assertEqual(easyblock, correct_easyblock) expected = "Toy C program, 100% toy." - self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], expected) + self.assertEqual(fetch_parameters_from_easyconfig(TOY_EC_TXT, ['description'])[0], expected) res = fetch_parameters_from_easyconfig("easyblock = 'ConfigureMake' # test comment", ['easyblock']) self.assertEqual(res, ['ConfigureMake']) @@ -1903,32 +1888,30 @@ def test_easyconfig_paths(self): def test_toolchain_inspection(self): """Test whether available toolchain inspection functionality is working.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) - ec = EasyConfig(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) tc_compilers = ['/'.join([x['name'], x['version']]) for x in det_toolchain_compilers(ec)] self.assertEqual(tc_compilers, ['GCC/6.4.0-2.28']) self.assertEqual(det_toolchain_mpi(ec)['name'], 'OpenMPI') - ec = EasyConfig(os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) tc_comps = det_toolchain_compilers(ec) expected = ['GCC/6.4.0-2.28'] self.assertEqual(['/'.join([x['name'], x['version'] + x['versionsuffix']]) for x in tc_comps], expected) self.assertEqual(det_toolchain_mpi(ec), None) - ec = EasyConfig(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) self.assertEqual(det_toolchain_compilers(ec), None) self.assertEqual(det_toolchain_mpi(ec), None) def test_filter_deps(self): """Test filtered dependencies.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') ec = EasyConfig(ec_file) self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) @@ -1953,13 +1936,13 @@ def test_filter_deps(self): build_options = { 'external_modules_metadata': ConfigObj(), 'minimal_toolchains': True, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) ec_file = os.path.join(self.test_prefix, 'test.eb') - shutil.copy2(os.path.join(test_ecs_dir, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb'), ec_file) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb'), ec_file) ec_txt = read_file(ec_file) ec_txt = ec_txt.replace('hwloc', 'deptobefiltered') @@ -1976,8 +1959,7 @@ def test_filter_deps(self): def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) replaced_parameters = { 'license': ('license_file', '2.0'), 'makeopts': ('buildopts', '2.0'), @@ -1996,10 +1978,7 @@ def foo(key): def test_alternative_easyconfig_parameters(self): """Test handling of alternative easyconfig parameters.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds') test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class') @@ -2045,8 +2024,7 @@ def test_deprecated_easyconfig_parameters(self): self.allow_deprecated_behaviour() init_config() - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + ec = EasyConfig(TOY_EC) easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = { 'foobar': ('barfoo', '0.0'), # deprecated since forever @@ -2097,9 +2075,8 @@ def foo(key): self.assertEqual(ec_params, expected) # try parsing an easyconfig file that defines a deprecated easyconfig parameter - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec)) + write_file(test_ec, TOY_EC_TXT) write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True) with self.mocked_stdout_stderr(): @@ -2130,8 +2107,7 @@ def set_ec_key(key): def test_external_dependencies(self): """Test specifying external (build) dependencies.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')) + ectxt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb')) toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') # just specify some of the test modules we ship, doesn't matter where they come from @@ -2385,10 +2361,6 @@ def test_external_dependencies(self): def test_external_dependencies_templates(self): """Test use of templates for dependencies marked as external modules.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ectxt = read_file(toy_ec) - extra_ectxt = '\n'.join([ "versionsuffix = '-Python-%(pyver)s-Perl-%(perlshortver)s'", '', @@ -2398,7 +2370,7 @@ def test_external_dependencies_templates(self): "]", ]) test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, toy_ectxt + '\n' + extra_ectxt) + write_file(test_ec, TOY_EC_TXT + '\n' + extra_ectxt) # put metadata in place so templates can be defined metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') @@ -2437,8 +2409,7 @@ def test_external_dependencies_templates(self): def test_update(self): """Test use of update() method for EasyConfig instances.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ebfile = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ebfile = TOY_EC ec = EasyConfig(toy_ebfile) # for string values: append @@ -2481,8 +2452,7 @@ def test_update(self): def test_hide_hidden_deps(self): """Test use of --hide-deps on hiddendependencies.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = TEST_ECS_DIR / 'g' / 'gzip' / 'gzip-1.4-GCC-4.6.3.eb' ec = EasyConfig(ec_file) self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') self.assertEqual(ec['dependencies'][0]['full_mod_name'], 'toy/.0.0-deps') @@ -2588,10 +2558,9 @@ def subtest_quote_py_str(val): def test_dump(self): """Test EasyConfig's dump() method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -2605,7 +2574,7 @@ def test_dump(self): for ecfile in ecfiles: test_ec = os.path.join(self.test_prefix, 'test.eb') - ec = EasyConfig(os.path.join(test_ecs_dir, ecfile)) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, ecfile)) with ec.disable_templating(): ecdict = ec.asdict() ec.dump(test_ec) @@ -2676,10 +2645,9 @@ def test_dump(self): def test_toolchain_hierarchy_aware_dump(self): """Test that EasyConfig's dump() method is aware of the toolchain hierarchy.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'check_osdeps': False, - 'robot_path': [test_ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -3283,16 +3251,15 @@ def test_dep_graph(self): print("Skipping test_dep_graph, since graphviz is not available") return - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, } init_config(build_options=build_options) - ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') ec_files = [(ec_file, False)] ecs, _ = parse_easyconfigs(ec_files) @@ -3326,20 +3293,16 @@ def test_dep_graph_multi_deps(self): print("Skipping test_dep_graph_multi_deps, since graphviz is not available") return - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, } init_config(build_options=build_options) - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = toy_ec_txt + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" + test_ec_txt = TOY_EC_TXT + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" write_file(test_ec, test_ec_txt) ec_files = [(test_ec, False)] @@ -3379,15 +3342,14 @@ def test_ActiveMNS_det_full_module_name(self): } init_config(build_options=build_options) - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + ec_file = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-deps.eb' ec = EasyConfig(ec_file) self.assertEqual(ActiveMNS().det_full_module_name(ec), 'toy/0.0-deps') self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][0]), 'intel/2018a') self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][1]), 'GCC/6.4.0-2.28') - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ec = EasyConfig(ec_file) hiddendep = ec['hiddendependencies'][0] self.assertEqual(ActiveMNS().det_full_module_name(hiddendep), 'toy/.0.0-deps') @@ -3395,46 +3357,45 @@ def test_ActiveMNS_det_full_module_name(self): def test_find_related_easyconfigs(self): """Test find_related_easyconfigs function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') ec = EasyConfig(ec_file) # exact match: GCC-4.6.3.eb - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['GCC-4.6.3.eb']) # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match ec['version'] = '4.6.1' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['GCC-4.6.4.eb', 'GCC-4.6.3.eb']) # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match ec['version'] = '4.5.0' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] expected = ['GCC-4.9.2.eb', 'GCC-4.8.3.eb', 'GCC-4.8.2.eb', 'GCC-4.6.4.eb', 'GCC-4.6.3.eb'] self.assertEqual(res, expected) - ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') ec = EasyConfig(ec_file) # exact match - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-deps.eb']) # tweak toolchain name/version and versionsuffix => closest match with same toolchain name is found ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} ec['versionsuffix'] = '-foobar' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb']) # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) ec['versionsuffix'] = '-deps' - res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + res = [os.path.basename(x) for x in find_related_easyconfigs(TEST_ECS_DIR, ec)] self.assertEqual(res, ['toy-0.0-deps.eb']) # no matches for unknown software name ec['name'] = 'nosuchsoftware' - self.assertEqual(find_related_easyconfigs(test_easyconfigs, ec), []) + self.assertEqual(find_related_easyconfigs(TEST_ECS_DIR, ec), []) # no problem with special characters in software name ec['name'] = 'nosuchsoftware++' @@ -3445,8 +3406,7 @@ def test_find_related_easyconfigs(self): def test_modaltsoftname(self): """Test specifying an alternative name for the software name, to use when determining module name.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + ec_file = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-deps.eb' ectxt = read_file(ec_file) modified_ec_file = os.path.join(self.test_prefix, os.path.basename(ec_file)) write_file(modified_ec_file, ectxt + "\nmodaltsoftname = 'notreallyatoy'") @@ -3458,15 +3418,14 @@ def test_modaltsoftname(self): def test_software_license(self): """Tests related to software_license easyconfig parameter.""" # default: None - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC ec = EasyConfig(ec_file) ec.validate_license() self.assertEqual(ec['software_license'], None) self.assertEqual(ec.software_license, None) # specified software license gets handled correctly - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') ec = EasyConfig(ec_file) ec.validate_license() # constant GPLv3 is resolved as string @@ -3481,8 +3440,7 @@ def test_software_license(self): def test_param_value_type_checking(self): """Test value tupe checking of easyconfig parameters.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb') + ec_file = TEST_ECS_DIR / 'g' / 'gzip' / 'gzip-1.4-broken.eb' # version parameter has values of wrong type in this broken easyconfig error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*" self.assertErrorRegex(EasyBuildError, error_msg_pattern, EasyConfig, ec_file, auto_convert_value_types=False) @@ -3495,8 +3453,7 @@ def test_copy(self): """Test copy method of EasyConfig object.""" init_config(build_options={'silent': True}) - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec1 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) + ec1 = EasyConfig(TOY_EC) # inject fake template value, just to check whether they are copied over too ec1.template_values['pyshortver'] = '3.7' @@ -3511,9 +3468,8 @@ def test_copy(self): def test_eq_hash(self): """Test comparing two EasyConfig instances.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec1 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) - ec2 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) + ec1 = EasyConfig(TOY_EC) + ec2 = EasyConfig(TOY_EC) # different instances, same parsed easyconfig self.assertIsNot(ec1, ec2) @@ -3525,14 +3481,13 @@ def test_eq_hash(self): self.assertEqual(hash(ec1), hash(ec2)) # other parsed easyconfig is not equal - ec3 = EasyConfig(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb')) + ec3 = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb')) self.assertFalse(ec1 == ec3) self.assertTrue(ec1 != ec3) def test_copy_easyconfigs(self): """Test copy_easyconfigs function.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') target_dir = os.path.join(self.test_prefix, 'copied_ecs') # easybuild/easyconfigs subdir is expected to exist @@ -3555,7 +3510,7 @@ def test_copy_easyconfigs(self): ecs_to_copy = [] for (src_ec, target_ec) in test_ecs: ecs_to_copy.append(os.path.join(self.test_prefix, target_ec)) - shutil.copy2(os.path.join(test_ecs_dir, src_ec), ecs_to_copy[-1]) + shutil.copy2(os.path.join(TEST_ECS_DIR, src_ec), ecs_to_copy[-1]) res = copy_easyconfigs(ecs_to_copy, target_dir) self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'new_file_in_existing_folder', @@ -3575,7 +3530,7 @@ def test_copy_easyconfigs(self): # create test easyconfig that includes comments & build stats, just like an archived easyconfig toy_ec = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'), toy_ec) + copy_file(TOY_EC, toy_ec) toy_ec_txt = read_file(toy_ec) toy_ec_txt = '\n'.join([ "# Built with EasyBuild version 3.1.2 on 2017-04-25_21-35-15", @@ -3618,8 +3573,7 @@ def test_copy_easyconfigs(self): def test_template_constant_dict(self): """Test template_constant_dict function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) arch_regex = re.compile('^[a-z0-9_]+$') @@ -3673,7 +3627,7 @@ def test_template_constant_dict(self): res.pop('arch') self.assertEqual(res, expected) - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') toy_ec_txt = read_file(toy_ec) # fiddle with version to check version_minor template ('0' should be retained) @@ -3803,10 +3757,8 @@ def test_template_constant_dict(self): def test_parse_deps_templates(self): """Test whether handling of templates defined by dependencies is done correctly.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - pyec = os.path.join(self.test_prefix, 'Python-2.7.10-foss-2018a.eb') - shutil.copy2(os.path.join(test_ecs, 'p', 'Python', 'Python-2.7.10-intel-2018a.eb'), pyec) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'p', 'Python', 'Python-2.7.10-intel-2018a.eb'), pyec) write_file(pyec, "\ntoolchain = {'name': 'foss', 'version': '2018a'}", append=True) ec_txt = '\n'.join([ @@ -3836,7 +3788,7 @@ def test_parse_deps_templates(self): build_options = { 'external_modules_metadata': ConfigObj(), - 'robot_path': [test_ecs, self.test_prefix], + 'robot_path': [TEST_ECS_DIR, self.test_prefix], 'valid_module_classes': module_classes(), 'validate': False, } @@ -3856,8 +3808,7 @@ def test_parse_deps_templates(self): def test_hidden_toolchain(self): """Test hiding of toolchain via easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_txt = read_file(os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')) + ec_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')) new_tc = "toolchain = {'name': 'GCC', 'version': '4.9.2', 'hidden': True}" ec_txt = re.sub("toolchain = .*", new_tc, ec_txt, re.M) @@ -3879,16 +3830,14 @@ def test_categorize_files_by_type(self): self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, categorize_files_by_type([])) - test_dir = os.path.dirname(os.path.abspath(__file__)) - test_ecs_dir = os.path.join(test_dir, 'easyconfigs') toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) - easyblocks_dir = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks') + easyblocks_dir = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') configuremake = os.path.join(easyblocks_dir, 'generic', 'configuremake.py') toy_easyblock = os.path.join(easyblocks_dir, 't', 'toy.py') - gzip_ec = os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') paths = [ 'bzip2-1.0.6.eb', toy_easyblock, @@ -4040,8 +3989,7 @@ def test_det_subtoolchain_version(self): def test_verify_easyconfig_filename(self): """Test verify_easyconfig_filename function""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_name = os.path.basename(toy_ec) specs = { 'name': 'toy', @@ -4094,16 +4042,14 @@ def test_get_paths_for(self): path.append(subdir) os.environ['PATH'] = os.pathsep.join(path) - top_dir = os.path.dirname(os.path.abspath(__file__)) mkdir(os.path.join(self.test_prefix, 'easybuild')) - test_ecs = os.path.join(top_dir, 'easyconfigs') - symlink(test_ecs, os.path.join(self.test_prefix, 'easybuild', 'easyconfigs')) + symlink(TEST_ECS_DIR, os.path.join(self.test_prefix, 'easybuild', 'easyconfigs')) # temporarily mock stderr to avoid printed warning (because 'eb' is not available via $PATH) with self.mocked_stderr(): # locations listed in 'robot_path' named argument are taken into account res = get_paths_for(subdir='easyconfigs', robot_path=[self.test_prefix]) - self.assertTrue(os.path.samefile(test_ecs, res[0])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[0])) # Can't have EB_SCRIPT_PATH set (for some of) these tests env_eb_script_path = os.getenv('EB_SCRIPT_PATH') @@ -4116,7 +4062,7 @@ def test_get_paths_for(self): os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), orig_path) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[-1])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[-1])) # also works when 'eb' resides in a symlinked location altbin = os.path.join(self.test_prefix, 'some', 'other', 'symlinked', 'bin') @@ -4124,7 +4070,7 @@ def test_get_paths_for(self): symlink(os.path.join(self.test_prefix, 'bin'), altbin) os.environ['PATH'] = '%s:%s' % (altbin, orig_path) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[-1])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[-1])) # Restore (temporarily) EB_SCRIPT_PATH value if set originally if env_eb_script_path: @@ -4134,7 +4080,7 @@ def test_get_paths_for(self): os.environ['PATH'] = orig_path sys.path.insert(0, self.test_prefix) res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.samefile(test_ecs, res[0])) + self.assertTrue(os.path.samefile(TEST_ECS_DIR, res[0])) # put mock 'eb' back in $PATH os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'bin'), orig_path) @@ -4194,10 +4140,9 @@ def test_get_module_path(self): def test_not_an_easyconfig(self): """Test error reporting when a file that's not actually an easyconfig file is provided.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) # run test on an easyconfig file that was downloaded using wget using a non-raw GitHub URL # cfr. https://github.com/easybuilders/easybuild-framework/issues/2383 - not_an_ec = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'not_an_easyconfig.eb') + not_an_ec = TEST_DIR / 'sandbox' / 'not_an_easyconfig.eb' # from Python 3.10 onwards: invalid decimal literal # older Python versions: invalid syntax @@ -4206,9 +4151,7 @@ def test_not_an_easyconfig(self): def test_check_sha256_checksums(self): """Test for check_sha256_checksums function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT checksums_regex = re.compile(r'^checksums = \[\[(.|\n)*\]\]', re.M) @@ -4256,7 +4199,7 @@ def test_check_sha256_checksums(self): self.assertEqual(check_sha256_checksums(ecs), []) # also test toy easyconfig with extensions, for which some checksums are missing - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ecs, _ = parse_easyconfigs([(toy_ec, False)]) ecs = [ec['ec'] for ec in ecs] @@ -4295,8 +4238,7 @@ def test_check_sha256_checksums(self): def test_deprecated(self): """Test use of 'deprecated' easyconfig parameter.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_txt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')) + toy_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, toy_ec_txt + "\ndeprecated = 'this is just a test'") @@ -4311,8 +4253,7 @@ def test_deprecated(self): def test_deprecated_toolchain(self): """Test use of deprecated toolchain""" - topdir = os.path.dirname(os.path.abspath(__file__)) - deprecated_toolchain_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb') + deprecated_toolchain_ec = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0-gompi-2018a.eb' init_config(build_options={'silence_deprecation_warnings': [], 'unit_testing_mode': False}) error_pattern = r"toolchain 'gompi/2018a' is marked as deprecated \(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, deprecated_toolchain_ec) @@ -4326,7 +4267,6 @@ def test_deprecated_toolchain(self): def test_filename(self): """Test filename method of EasyConfig class.""" init_config(build_options={'silent': True}) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') test_ecs = [ os.path.join('g', 'GCC', 'GCC-4.6.4.eb'), @@ -4337,14 +4277,13 @@ def test_filename(self): os.path.join('t', 'toy', 'toy-0.0-deps.eb'), ] for test_ec in test_ecs: - test_ec = os.path.join(test_ecs_dir, test_ec) + test_ec = os.path.join(TEST_ECS_DIR, test_ec) ec = EasyConfig(test_ec) self.assertTrue(ec.filename(), os.path.basename(test_ec)) def test_get_ref(self): """Test get_ref method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-iter.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-iter.eb')) # without using get_ref, we get a (templated) copy rather than the original value sources = ec['sources'] @@ -4375,9 +4314,8 @@ def test_get_ref(self): def test_multi_deps(self): """Test handling of multi_deps easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec = TOY_EC + toy_ec_txt = TOY_EC_TXT ec = EasyConfig(toy_ec) self.assertEqual(ec['builddependencies'], []) @@ -4450,9 +4388,7 @@ def test_multi_deps(self): def test_multi_deps_templated_builddeps(self): """Test effect of multi_deps on builddependencies w.r.t. resolving templates like %(pyver)s.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = toy_ec_txt + "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}" @@ -4499,12 +4435,8 @@ def test_multi_deps_templated_builddeps(self): def test_iter_builddeps_templates(self): """Test whether iterative builddependencies are taken into account to define *ver and *shortver templates.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = toy_ec_txt + "\nmulti_deps = {'Python': ['2.7.15', '3.6.6']}" + test_ec_txt = TOY_EC_TXT + "\nmulti_deps = {'Python': ['2.7.15', '3.6.6']}" # inject extension that uses %(pyshortver)s, to check whether the template value is properly resolved test_ec_txt += '\n'.join([ @@ -4560,21 +4492,16 @@ def test_iter_builddeps_templates(self): def test_fix_deprecated_easyconfigs(self): """Test fix_deprecated_easyconfigs function.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') # need to allow triggering deprecated behaviour, since that's exactly what we're fixing... self.allow_deprecated_behaviour() - test_ectxt = toy_ec_txt # inject local variables with names that need to be tweaked (or not for single-letter ones) regex = re.compile('^(sanity_check_paths)', re.M) # purposely define configopts via local variable 'foo', which has value that also contains 'foo' substring; # that way, we can check whether only the 'foo' variable name is replaced with 'local_foo' - test_ectxt = regex.sub(r'foo = "--foobar --barfoo --barfoobaz"\nconfigopts = foo\n\n\1', toy_ec_txt) + test_ectxt = regex.sub(r'foo = "--foobar --barfoo --barfoobaz"\nconfigopts = foo\n\n\1', TOY_EC_TXT) regex = re.compile(r'^(toolchain\s*=.*)$', re.M) test_ectxt = regex.sub(r'\1\n\nsome_list = [x + "1" for x in ["one", "two", "three"]]', test_ectxt) @@ -4969,8 +4896,7 @@ def test_det_copy_ec_specs(self): def test_recursive_module_unload(self): """Test use of recursive_module_unload easyconfig parameter.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) @@ -5086,8 +5012,7 @@ def test_pure_ec(self): Test whether we can get a 'pure' view on the easyconfig file, which correctly reflects what's defined in the easyconfig file. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + toy_ec = EasyConfig(TOY_EC) ec_dict = toy_ec.parser.get_config_dict() self.assertEqual(ec_dict.get('version'), '0.0') @@ -5111,11 +5036,9 @@ def test_easyconfig_import(self): """ Test parsing of an easyconfig file that includes import statements. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "import os", "local_test = os.getenv('TEST_TOY')", @@ -5250,11 +5173,8 @@ def test_get_amdgcn_cc_template_value(self): def test_count_files(self): """Tests for EasyConfig.count_files method.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - foss = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') - toy = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_exts = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + foss = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + toy_exts = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # no sources or patches for toolchain => 0 foss_ec = EasyConfig(foss) @@ -5262,7 +5182,7 @@ def test_count_files(self): self.assertEqual(foss_ec['patches'], []) self.assertEqual(foss_ec.count_files(), 0) # 1 source + 2 patches => 3 - toy_ec = EasyConfig(toy) + toy_ec = EasyConfig(TOY_EC) self.assertEqual(len(toy_ec['sources']), 1) self.assertEqual(len(toy_ec['patches']), 2) self.assertEqual(toy_ec['exts_list'], []) @@ -5310,13 +5230,11 @@ def test_easyconfigs_caches(self): """ Test whether easyconfigs caches work as intended. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - libtoy_ec = os.path.join(test_ecs_dir, 'l', 'libtoy', 'libtoy-0.0.eb') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + libtoy_ec = os.path.join(TEST_ECS_DIR, 'l', 'libtoy', 'libtoy-0.0.eb') copy_file(libtoy_ec, self.test_prefix) - copy_file(toy_ec, self.test_prefix) + copy_file(TOY_EC, self.test_prefix) libtoy_ec = os.path.join(self.test_prefix, os.path.basename(libtoy_ec)) - toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) + toy_ec = os.path.join(self.test_prefix, os.path.basename(TOY_EC)) ec1 = process_easyconfig(toy_ec)[0] self.assertEqual(ec1['ec'].name, 'toy') @@ -5378,10 +5296,8 @@ def test_templates(self): """ Test use of template values like %(version)s """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\ndescription = "name: %(name)s, version: %(version)s"' test_ec = os.path.join(self.test_prefix, 'test.eb') diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 8125c941d4..c8d2730f21 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -27,8 +27,8 @@ @author: Stijn De Weirdt (Ghent University) """ -import os import sys +from test.framework import TEST_DIR, TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -40,15 +40,14 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file - -TESTDIRBASE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') +EASYCONFIGS_DIR = TEST_DIR / 'easyconfigs' class EasyConfigParserTest(EnhancedTestCase): """Test the parser""" def test_v10(self): - ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb')) + ecp = EasyConfigParser(EASYCONFIGS_DIR / 'v1.0' / 'g' / 'GCC' / 'GCC-4.6.3.eb') self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0')) @@ -72,7 +71,7 @@ def test_v20(self): orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True - fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb') + fn = EASYCONFIGS_DIR / 'v2.0' / 'GCC.eb' ecp = EasyConfigParser(fn) formatter = ecp._formatter @@ -105,7 +104,7 @@ def test_v20_extra(self): orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True - fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb') + fn = TEST_DIR / 'easyconfigs' / 'v2.0' / 'doesnotexist.eb' ecp = EasyConfigParser(fn) formatter = ecp._formatter @@ -124,7 +123,7 @@ def test_v20_deps(self): orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True - fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb') + fn = EASYCONFIGS_DIR / 'v2.0' / 'libpng.eb' ecp = EasyConfigParser(fn) ec = ecp.get_config_dict() @@ -139,7 +138,7 @@ def test_v20_deps(self): self.assertEqual(deps[0].name(), 'zlib') self.assertEqual(deps[0].version(), '1.2.5') - fn = os.path.join(TESTDIRBASE, 'v2.0', 'foss.eb') + fn = EASYCONFIGS_DIR / 'v2.0' / 'foss.eb' ecp = EasyConfigParser(fn) ec = ecp.get_config_dict() @@ -167,9 +166,9 @@ def test_v20_deps(self): def test_raw(self): """Test passing of raw contents to EasyConfigParser.""" - ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb') + ec_file1 = EASYCONFIGS_DIR / 'v1.0' / 'g' / 'GCC' / 'GCC-4.6.3.eb' ec_txt1 = read_file(ec_file1) - ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + ec_file2 = EASYCONFIGS_DIR / 'v1.0' / 'g' / 'gzip' / 'gzip-1.5-foss-2018a.eb' ec_txt2 = read_file(ec_file2) ecparser = EasyConfigParser(ec_file1) @@ -211,7 +210,7 @@ def test_easyconfig_constants(self): def test_check_value_types(self): """Test checking of easyconfig parameter value types.""" - test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb') + test_ec = TEST_ECS_DIR / 'g' / 'gzip' / 'gzip-1.4-broken.eb' error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*" ecp = EasyConfigParser(test_ec, auto_convert_value_types=False) self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 923c3f1fb4..7abb9b56d3 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -39,6 +39,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework import TEST_DIR class EasyStackTest(EnhancedTestCase): @@ -60,15 +61,13 @@ def tearDown(self): def test_easystack_basic(self): """Test for basic easystack files.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystacks = [ 'test_easystack_basic.yaml', 'test_easystack_basic_dict.yaml', 'test_easystack_easyconfigs_with_eb_ext.yaml', ] for fn in test_easystacks: - test_easystack = os.path.join(topdir, 'easystacks', fn) + test_easystack = os.path.join(TEST_DIR, 'easystacks', fn) easystack = parse_easystack(test_easystack) expected = [ @@ -83,8 +82,7 @@ def test_easystack_basic(self): def test_easystack_easyconfigs_dict(self): """Test for easystack file where easyconfigs item is parsed as a dict, because easyconfig names are not prefixed by dashes""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_dict.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_easyconfigs_dict.yaml') error_pattern = r"Found dict value for 'easyconfigs' in .* should be list.\nMake sure you use '-' to create .*" self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) @@ -92,16 +90,14 @@ def test_easystack_easyconfigs_dict(self): def test_easystack_easyconfigs_str(self): """Test for easystack file where easyconfigs item is parsed as a dict, because easyconfig names are not prefixed by dashes""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_str.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_easyconfigs_str.yaml') error_pattern = r"Found str value for 'easyconfigs' in .* should be list.\nMake sure you use '-' to create .*" self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_easyconfig_opts(self): """Test an easystack file using the 'easyconfigs' key, with additonal options for some easyconfigs""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_opts.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_easyconfigs_opts.yaml') easystack = parse_easystack(test_easystack) expected_tuples = [ @@ -114,16 +110,14 @@ def test_easystack_easyconfig_opts(self): def test_easystack_invalid_key(self): """Test easystack files with invalid key at the same level as the 'options' key""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_invalid_key.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_invalid_key.yaml') error_pattern = r"Found one or more invalid keys for .* \(only 'options' supported\).*" self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_invalid_key2(self): """Test easystack files with invalid key at the same level as the key that names the easyconfig""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_invalid_key2.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_invalid_key2.yaml') error_pattern = r"expected a dictionary with one key \(the EasyConfig name\), " error_pattern += r"instead found keys: .*, invalid_key" @@ -177,8 +171,7 @@ def test_easystack_restore_env_after_each_build(self): def test_missing_easyconfigs_key(self): """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_missing_easyconfigs_key.yaml') + test_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_missing_easyconfigs_key.yaml') error_pattern = r"Top-level key 'easyconfigs' missing in easystack file %s" % test_easystack self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 18130d166a..88c83c8c27 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -57,6 +57,8 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import LINUX, get_os_type +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT + class FileToolsTest(EnhancedTestCase): """ Testcase for filetools module """ @@ -561,8 +563,7 @@ def test_download_file(self): fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_source_dir = os.path.join(test_dir, 'sandbox', 'sources', 'toy') + toy_source_dir = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') source_url = 'file://%s/%s' % (toy_source_dir, fn) with self.mocked_stdout_stderr(): res = ft.download_file(fn, source_url, target_location) @@ -587,11 +588,11 @@ def test_download_file(self): # non-existing files result in None return value with self.mocked_stdout_stderr(): - self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) + self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % TEST_DIR, target_location), None) # install broken proxy handler for opening local files # this should make urlopen use this broken proxy for downloading from a file:// URL - proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) + proxy_handler = request.ProxyHandler({'file': 'file://%s/nosuchfile' % TEST_DIR}) request.install_opener(request.build_opener(proxy_handler)) # for Python 3.14+, we need to make sure that proxy and original URL are using different protocol, @@ -727,8 +728,7 @@ def fake_urllib_open(url, *args, **kwargs): return self.orig_filetools_std_urllib_urlopen(url, *args, **kwargs) fn = 'toy-0.0.eb' - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_dir = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy') + toy_dir = os.path.join(TEST_ECS_DIR, 't', 'toy') url = 'file://%s/%s' % (toy_dir, fn) ft.std_urllib.urlopen = fake_urllib_open @@ -793,8 +793,7 @@ def test_download_file_fallback_source_urls(self): """ fn = 'toy-0.0.eb' - test_dir = os.path.abspath(os.path.dirname(__file__)) - toy_dir = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy') + toy_dir = os.path.join(TEST_ECS_DIR, 't', 'toy') correct_url = f'file://{toy_dir}/' wrong_url = f'file://{self.test_prefix}/easyconfigs/' @@ -1102,7 +1101,7 @@ def test_is_binary(self): def test_det_patched_files(self): """Test det_patched_files function.""" toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - pf = os.path.join(os.path.dirname(__file__), 'sandbox', 'sources', 'toy', toy_patch_fn) + pf = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) self.assertEqual(ft.det_patched_files(pf), ['b/toy-0.0/toy.source']) self.assertEqual(ft.det_patched_files(pf, omit_ab_prefix=True), ['toy-0.0/toy.source']) @@ -1304,15 +1303,13 @@ def test_move_logs(self): def test_multidiff(self): """Test multidiff function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') other_toy_ecs = [ - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb'), - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), ] # default (colored) - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - lines = multidiff(toy_ec, other_toy_ecs).split('\n') + lines = multidiff(TOY_EC, other_toy_ecs).split('\n') expected = "Comparing \x1b[0;35mtoy-0.0.eb\x1b[0m with toy-0.0-deps.eb, toy-0.0-gompi-2018a-test.eb" red = "\x1b[0;41m" @@ -1342,7 +1339,7 @@ def test_multidiff(self): self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) self.assertEqual(lines[-1], "=====") - lines = multidiff(toy_ec, other_toy_ecs, colored=False).split('\n') + lines = multidiff(TOY_EC, other_toy_ecs, colored=False).split('\n') self.assertEqual(lines[0], "Comparing toy-0.0.eb with toy-0.0-deps.eb, toy-0.0-gompi-2018a-test.eb") self.assertEqual(lines[1], "=====") @@ -1801,10 +1798,9 @@ def test_find_flexlm_license(self): def test_is_patch_file(self): """Test for is_patch_file() function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - self.assertFalse(ft.is_patch_file(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))) + self.assertFalse(ft.is_patch_file(TOY_EC)) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - self.assertTrue(ft.is_patch_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn))) + self.assertTrue(ft.is_patch_file(os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn))) def test_is_alt_pypi_url(self): """Test is_alt_pypi_url() function.""" @@ -1905,12 +1901,11 @@ def test_create_patch_info(self): def test_apply_patch(self): """ Test apply_patch """ - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_tar_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_tar_gz = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) for with_backup in (True, False): update_build_option('backup_patched_files', with_backup) @@ -1929,7 +1924,7 @@ def test_apply_patch(self): self.assertNotExists(backup_file) # This patch is dependent on the previous one - toy_patch_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz') + toy_patch_gz = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz') with self.mocked_stdout_stderr(): self.assertTrue(ft.apply_patch(toy_patch_gz, path)) patched_gz = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source')) @@ -2003,12 +1998,10 @@ def test_apply_patch(self): def test_copy_file(self): """Test copy_file function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(self.test_prefix, 'toy.eb') - ft.copy_file(toy_ec, target_path) + ft.copy_file(TOY_EC, target_path) self.assertExists(target_path) - self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) + self.assertEqual(ft.read_file(target_path), TOY_EC_TXT) # Make sure it doesn't fail if path is a symlink and target_path is a dir toy_link_fn = 'toy-link-0.0.eb' @@ -2032,7 +2025,7 @@ def test_copy_file(self): ft.remove_file(copied_file) # clean error when trying to copy a directory with copy_file - src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy') + src, target = os.path.dirname(TOY_EC), os.path.join(self.test_prefix, 'toy') # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist" error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target) @@ -2082,7 +2075,7 @@ def test_copy_file(self): self.assertNotExists(target_path) with self.mocked_stdout(): - ft.copy_file(toy_ec, target_path) + ft.copy_file(TOY_EC, target_path) txt = self.get_stdout() self.assertNotExists(target_path) @@ -2090,11 +2083,11 @@ def test_copy_file(self): # forced copy, even in dry run mode with self.mocked_stdout(): - ft.copy_file(toy_ec, target_path, force_in_dry_run=True) + ft.copy_file(TOY_EC, target_path, force_in_dry_run=True) txt = self.get_stdout() self.assertExists(target_path) - self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) + self.assertEqual(ft.read_file(target_path), TOY_EC_TXT) self.assertEqual(txt, '') # Test that a non-existing file raises an exception @@ -2155,18 +2148,15 @@ def test_copy_file_xattr(self): def test_copy_files(self): """Test copy_files function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = ft.read_file(toy_ec) - bzip2_ec = os.path.join(test_ecs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') + bzip2_ec = os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') bzip2_ec_txt = ft.read_file(bzip2_ec) # copying a single file to a non-existing directory target_dir = os.path.join(self.test_prefix, 'target_dir1') - ft.copy_files([toy_ec], target_dir) + ft.copy_files([TOY_EC], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) # copying a single file to an existing directory ft.copy_files([bzip2_ec], target_dir) @@ -2176,10 +2166,10 @@ def test_copy_files(self): # copying multiple files to a non-existing directory target_dir = os.path.join(self.test_prefix, 'target_dir_multiple') - ft.copy_files([toy_ec, bzip2_ec], target_dir) + ft.copy_files([TOY_EC, bzip2_ec], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) copied_bzip2_ec = os.path.join(target_dir, 'bzip2-1.0.6-GCC-4.9.2.eb') self.assertExists(copied_bzip2_ec) self.assertEqual(ft.read_file(copied_bzip2_ec), bzip2_ec_txt) @@ -2197,10 +2187,10 @@ def test_copy_files(self): # test special case: copying a single file to a file target via target_single_file=True target = os.path.join(self.test_prefix, 'target') self.assertNotExists(target) - ft.copy_files([toy_ec], target, target_single_file=True) + ft.copy_files([TOY_EC], target, target_single_file=True) self.assertExists(target) self.assertTrue(os.path.isfile(target)) - self.assertEqual(toy_ec_txt, ft.read_file(target)) + self.assertEqual(ft.read_file(target), TOY_EC_TXT) ft.remove_file(target) @@ -2208,27 +2198,27 @@ def test_copy_files(self): target = os.path.join(self.test_prefix, 'target_parent', 'target_subdir', 'target.txt') self.assertNotExists(target) self.assertNotExists(os.path.dirname(target)) - ft.copy_files([toy_ec], target, target_single_file=True) + ft.copy_files([TOY_EC], target, target_single_file=True) self.assertExists(target) self.assertTrue(os.path.isfile(target)) - self.assertEqual(toy_ec_txt, ft.read_file(target)) + self.assertEqual(ft.read_file(target), TOY_EC_TXT) ft.remove_file(target) # default behaviour is to copy single file list to target *directory* self.assertNotExists(target) - ft.copy_files([toy_ec], target) + ft.copy_files([TOY_EC], target) self.assertExists(target) self.assertTrue(os.path.isdir(target)) copied_toy_ec = os.path.join(target, 'toy-0.0.eb') self.assertExists(copied_toy_ec) - self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) + self.assertEqual(ft.read_file(copied_toy_ec), TOY_EC_TXT) ft.remove_dir(target) # test enabling verbose mode with self.mocked_stdout_stderr(): - ft.copy_files([toy_ec], target, verbose=True) + ft.copy_files([TOY_EC], target, verbose=True) stderr, stdout = self.get_stderr(), self.get_stdout() self.assertEqual(stderr, '') regex = re.compile(r"^1 file\(s\) copied to .*/target") @@ -2237,7 +2227,7 @@ def test_copy_files(self): ft.remove_dir(target) with self.mocked_stdout_stderr(): - ft.copy_files([toy_ec], target, target_single_file=True, verbose=True) + ft.copy_files([TOY_EC], target, target_single_file=True, verbose=True) stderr, stdout = self.get_stderr(), self.get_stdout() self.assertEqual(stderr, '') regex = re.compile(r"/.*/toy-0\.0\.eb copied to .*/target") @@ -2312,8 +2302,7 @@ def test_has_recursive_symlinks(self): def test_copy_dir(self): """Test copy_dir function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') + to_copy = os.path.join(TEST_ECS_DIR, 'g', 'GCC') target_dir = os.path.join(self.test_prefix, 'GCC') self.assertNotExists(target_dir) @@ -2419,12 +2408,10 @@ def ignore_func(_, names): def test_copy(self): """Test copy function.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - - toy_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_file = TOY_EC toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) - gcc_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) + gcc_dir = os.path.join(TEST_ECS_DIR, 'g', 'GCC') ft.copy([toy_file, gcc_dir, toy_patch], self.test_prefix) @@ -2506,8 +2493,7 @@ def test_extract_file(self): """Test extract_file""" cwd = os.getcwd() - testdir = os.path.dirname(os.path.abspath(__file__)) - toy_tarball = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') + toy_tarball = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) with self.mocked_stdout_stderr(): @@ -2708,23 +2694,20 @@ def test_clean_dir(self): def test_index_functions(self): """Test *_index functions.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # create_index checks whether specified path is an existing directory doesnotexist = os.path.join(self.test_prefix, 'doesnotexist') self.assertErrorRegex(EasyBuildError, "Specified path does not exist", ft.create_index, doesnotexist) - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, toy_ec) + self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, TOY_EC) # load_index just returns None if there is no index in specified directory self.assertEqual(ft.load_index(self.test_prefix), None) - num_files = len(glob.glob(test_ecs + '/**/*.*', recursive=True)) + num_files = len(list(TEST_ECS_DIR.rglob('*.*'))) # create index for test easyconfigs; # test with specified path with and without trailing '/'s - for path in [test_ecs, test_ecs + '/', test_ecs + '//']: + for path in [TEST_ECS_DIR, str(TEST_ECS_DIR), str(TEST_ECS_DIR) + '/', str(TEST_ECS_DIR) + '//']: index = ft.create_index(path) self.assertEqual(len(index), num_files) @@ -2741,7 +2724,7 @@ def test_index_functions(self): # set up some files to create actual index file for ecs_dir = os.path.join(self.test_prefix, 'easyconfigs') - ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir) + ft.copy_dir(os.path.join(TEST_ECS_DIR, 'g'), ecs_dir) # test dump_index function index_fp = ft.dump_index(ecs_dir) @@ -2824,10 +2807,9 @@ def test_index_functions(self): def test_search_file(self): """Test search_file function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # check for default semantics, test case-insensitivity - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True) self.assertEqual(var_defs, []) self.assertEqual(len(hits), 5) self.assertTrue(all(os.path.exists(p) for p in hits)) @@ -2838,16 +2820,16 @@ def test_search_file(self): self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching - var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) self.assertEqual(hits_case_sensitive, []) - var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([TEST_ECS_DIR], 'hwloc', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) self.assertEqual(hits_case_sensitive, hits) # check filename-only mode - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', 'hwloc-1.8-gcccuda-2018a.eb', @@ -2857,12 +2839,12 @@ def test_search_file(self): ]) # check specifying of ignored dirs - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, ignore_dirs=['hwloc']) self.assertEqual(var_defs + hits, []) # check short mode - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) - self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', silent=True, short=True) + self.assertEqual(var_defs, [('CFGS1', os.path.join(TEST_ECS_DIR, 'h', 'hwloc'))]) self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', '$CFGS1/hwloc-1.8-gcccuda-2018a.eb', '$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', @@ -2871,19 +2853,19 @@ def test_search_file(self): ]) # check terse mode (implies 'silent', overrides 'short') - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', terse=True, short=True) self.assertEqual(var_defs, []) expected = [ - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only - var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', 'hwloc-1.8-gcccuda-2018a.eb', @@ -2895,7 +2877,7 @@ def test_search_file(self): # patterns that include special characters + (or ++) shouldn't cause trouble # cfr. https://github.com/easybuilders/easybuild-framework/issues/2966 for pattern in ['netCDF-C++', 'foo.*bar', 'foo|bar']: - var_defs, hits = ft.search_file([test_ecs], pattern, terse=True, filename_only=True) + var_defs, hits = ft.search_file([TEST_ECS_DIR], pattern, terse=True, filename_only=True) self.assertEqual(var_defs, []) # no hits for any of these in test easyconfigs self.assertEqual(hits, []) @@ -2904,7 +2886,7 @@ def test_search_file(self): # to avoid accidental matches in other files already present (log files, etc.) ec_dir = tempfile.mkdtemp() test_ec = os.path.join(ec_dir, 'netCDF-C++-4.2-foss-2019a.eb') - ft.write_file(test_ec, ''), + ft.write_file(test_ec, '') for pattern in ['netCDF-C++', 'CDF', 'C++', '^netCDF']: var_defs, hits = ft.search_file([ec_dir], pattern, terse=True, filename_only=True) self.assertEqual(var_defs, [], msg='For pattern ' + pattern) @@ -2912,7 +2894,7 @@ def test_search_file(self): # check how simply invalid queries are handled for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']: - self.assertErrorRegex(EasyBuildError, "Invalid search query", ft.search_file, [test_ecs], pattern) + self.assertErrorRegex(EasyBuildError, "Invalid search query", ft.search_file, [TEST_ECS_DIR], pattern) def test_dir_contains_files(self): def makedirs_in_test(*paths): @@ -3564,8 +3546,7 @@ def foobar(): def test_get_easyblock_class_name(self): """Test for get_easyblock_class_name function.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') self.assertEqual(ft.get_easyblock_class_name(configuremake), 'ConfigureMake') @@ -3579,8 +3560,7 @@ def test_get_easyblock_class_name(self): def test_copy_easyblocks(self): """Test for copy_easyblocks function.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # easybuild/easyblocks subdirectory must exist in target directory error_pattern = "Could not find easybuild/easyblocks subdir in .*" @@ -3654,7 +3634,6 @@ def test_copy_framework_files(self): # create empty test/framework/modules.py, to check whether 'new' is set correctly in result ft.write_file(os.path.join(target_dir, 'test', 'framework', 'modules.py'), '') - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_files = [ os.path.join('easybuild', 'tools', 'filetools.py'), os.path.join('test', 'framework', 'modules.py'), @@ -3670,7 +3649,7 @@ def test_copy_framework_files(self): # setup.py is an important test case, since it has no parent directory # (it's straight in the easybuild-framework directory) setup_py = 'setup.py' - if os.path.exists(os.path.join(topdir, setup_py)): + if os.path.exists(os.path.join(REPO_ROOT, setup_py)): test_files.append(os.path.join(setup_py)) expected_entries.append(setup_py) expected_new.append(True) @@ -3679,7 +3658,7 @@ def test_copy_framework_files(self): # so we need to make sure that's the case here as well (may not be in workspace dir on Travis from example) framework_dir = os.path.join(self.test_prefix, 'easybuild-framework') for test_file in test_files: - ft.copy_file(os.path.join(topdir, test_file), os.path.join(framework_dir, test_file)) + ft.copy_file(os.path.join(REPO_ROOT, test_file), os.path.join(framework_dir, test_file)) test_paths = [os.path.join(framework_dir, f) for f in test_files] @@ -3690,7 +3669,7 @@ def test_copy_framework_files(self): self.assertEqual(sorted(res.keys()), ['new', 'paths_in_repo']) for idx, test_file in enumerate(test_files): - orig_path = os.path.join(topdir, test_file) + orig_path = os.path.join(REPO_ROOT, test_file) copied_path = os.path.join(target_dir, test_file) self.assertExists(copied_path) diff --git a/test/framework/github.py b/test/framework/github.py index 774ef3591f..d03dc08333 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -37,6 +37,7 @@ import textwrap import unittest from string import ascii_letters +from test.framework import TEST_DIR, TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from time import gmtime from unittest import TextTestRunner @@ -733,8 +734,6 @@ def test_github_find_easybuild_easyconfig(self): def test_github_find_patches(self): """ Test for find_software_name_for_patch """ - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_path = os.path.join(test_dir, 'easyconfigs') init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, @@ -745,7 +744,7 @@ def test_github_find_patches(self): 'validate': False, }) with self.mocked_stdout(): - ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [ec_path]) + ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [TEST_ECS_DIR]) txt = self.get_stdout() self.assertEqual(ec, 'toy') @@ -1148,12 +1147,10 @@ def test_github_det_pr_target_repo(self): # no files => return default target repo (None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), None) - test_dir = os.path.dirname(os.path.abspath(__file__)) - # easyconfigs/patches (incl. files to delete) => easyconfigs repo # this is solely based on filenames, actual files are not opened, except for the patch file which must exist toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(test_dir, 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) test_cases = [ ['toy.eb'], [toy_patch], @@ -1168,11 +1165,11 @@ def test_github_det_pr_target_repo(self): # if only Python files are involved, result is easyblocks or framework repo; # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! - github_py = os.path.join(test_dir, 'github.py') + github_py = os.path.join(TEST_DIR, 'github.py') - configuremake = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') + configuremake = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') self.assertExists(configuremake) - toy_eb = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertExists(toy_eb) self.assertEqual(build_option('pr_target_repo'), None) diff --git a/test/framework/include.py b/test/framework/include.py index 6be29381e4..fb77e9d21f 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -29,6 +29,7 @@ """ import os import sys +from test.framework import TEST_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -53,7 +54,7 @@ class IncludeTest(EnhancedTestCase): def test_include_easyblocks(self): """Test include_easyblocks().""" - test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # put a couple of custom easyblocks in place, to test myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks') @@ -119,7 +120,7 @@ def test_include_easyblocks(self): def test_include_easyblocks_priority(self): """Test whether easyblocks included via include_easyblocks() get priority over others.""" - test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # make sure that test 'foo' easyblock is there import easybuild.easyblocks.foo @@ -288,8 +289,7 @@ def test_is_software_specific_easyblock(self): self.assertErrorRegex(EasyBuildError, "No such file", is_software_specific_easyblock, '/no/such/easyblock.py') - testdir = os.path.dirname(os.path.abspath(__file__)) - test_easyblocks = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 'g', 'gcc.py'))) self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 't', 'toy.py'))) diff --git a/test/framework/lib.py b/test/framework/lib.py index 176c14221f..8817140c2f 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -33,6 +33,7 @@ import tempfile from unittest import TextTestRunner +from test.framework import TEST_MODULES_DIR from test.framework.utilities import TestLoaderFiltered # deliberately *not* using EnhancedTestCase from test.framework.utilities to avoid automatic configuration via setUp @@ -125,7 +126,7 @@ def test_modules_tool(self): self.configure() - test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_mods_path = os.path.join(TEST_MODULES_DIR) modtool = modules_tool() modtool.use(test_mods_path) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 5834fb731c..ba7cba5833 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -45,6 +45,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod from easybuild.tools.utilities import quote_str +from test.framework import TEST_MODULES_DIR, TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config @@ -57,8 +58,7 @@ def setUp(self): """Test setup.""" super().setUp() # find .eb file - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') + eb_path = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb') eb_full_path = find_full_path(eb_path) self.assertTrue(eb_full_path) @@ -801,9 +801,8 @@ def test_module_extensions(self): # check if extensions option is enabled and some module extensions are defined init_config(build_options={'module_extensions': True}) - test_dir = os.path.abspath(os.path.dirname(__file__)) - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR) + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -833,7 +832,7 @@ def test_module_extensions(self): self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc)) # check if the extensions is missing if there are no extensions - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -849,7 +848,7 @@ def test_module_extensions(self): # check if the extensions is missing if 'module_extensions' is disabled init_config(build_options={'module_extensions': False}) - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') ec = EasyConfig(test_ec) eb = EasyBlock(ec) @@ -1358,15 +1357,14 @@ def test_module_naming_scheme(self): all_stops = [x[0] for x in EasyBlock.get_steps()] init_config(build_options={'valid_stops': all_stops}) - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files] + ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(TEST_ECS_DIR) for fil in files] # keep only easyconfig files (there may be additional files like patches, checksums.json, etc.) ec_files = [x for x in ec_files if x.endswith('.eb')] build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1421,7 +1419,7 @@ def test_mns(): init_config(build_options=build_options) err_pattern = 'nosucheasyconfigparameteravailable' - ec_file = os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') self.assertErrorRegex(EasyBuildError, err_pattern, EasyConfig, ec_file) # test simple custom module naming scheme @@ -1438,7 +1436,7 @@ def test_mns(): } test_mns() - ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/2018a') # test module naming scheme using all available easyconfig parameters @@ -1485,7 +1483,7 @@ def test_mns(): # determine full module name self.assertEqual(ActiveMNS().det_full_module_name(dep_spec), ec2mod_map[dep_ec]) - ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True) self.assertEqual(ec.full_mod_name, ec2mod_map['gzip-1.5-foss-2018a.eb']) self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/e69469ac250145c9e814e5dde93f5fde6d80375d') @@ -1554,11 +1552,10 @@ def test_hierarchical_mns(self): """Test hierarchical module naming scheme.""" moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'] - ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, 'valid_module_classes': moduleclasses, @@ -1566,7 +1563,7 @@ def test_hierarchical_mns(self): def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): """Test whether active module naming scheme returns expected values.""" - ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0]) + ec = EasyConfig(glob.glob(os.path.join(TEST_ECS_DIR, '*', '*', ecfile))[0]) self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) @@ -1652,7 +1649,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, test_ec(ecfile, *mns_vals) # impi with dummy toolchain, which doesn't make sense in a hierarchical context - ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' @@ -1695,7 +1692,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, test_ec(ecfile, *mns_vals, init_modpaths=['Core/%s' % c for c in moduleclasses]) # impi with dummy toolchain, which doesn't make sense in a hierarchical context - ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb')) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedModuleNamingScheme' diff --git a/test/framework/modules.py b/test/framework/modules.py index 589df6ccd5..f257fa7c40 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -36,6 +36,7 @@ import shutil import stat import sys +from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -64,7 +65,7 @@ class ModulesTest(EnhancedTestCase): def init_testmods(self, test_modules_paths=None): """Initialize set of test modules for test.""" if test_modules_paths is None: - test_modules_paths = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))] + test_modules_paths = [os.path.abspath(os.path.join(TEST_MODULES_DIR))] self.reset_modulepath(test_modules_paths) # for Lmod, this test has to run first, to avoid that it fails; @@ -83,7 +84,7 @@ def test_long_module_path(self): # copy one of the test modules there gcc_mod_dir = os.path.join(long_mod_path, 'GCC') os.makedirs(gcc_mod_dir) - gcc_mod_path = os.path.join(os.path.dirname(__file__), 'modules', 'GCC', '4.6.3') + gcc_mod_path = os.path.join(TEST_MODULES_DIR, 'GCC', '4.6.3') copy_file(gcc_mod_path, gcc_mod_dir) # try and use long modules path @@ -97,8 +98,6 @@ def test_long_module_path(self): def test_run_module(self): """Test for ModulesTool.run_module method.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']: os.environ.pop(key, None) @@ -107,7 +106,7 @@ def test_run_module(self): self.modtool.run_module('load', 'GCC/6.4.0-2.28') self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') - self.reset_modulepath([os.path.join(testdir, 'modules')]) + self.reset_modulepath([os.path.join(TEST_MODULES_DIR)]) self.assertNotIn('EBROOTGCC', os.environ) self.modtool.run_module(['load', 'GCC/6.4.0-2.28']) @@ -266,7 +265,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['OpenMPI'], maybe_partial=False, skip_avail=True), [False]) # exist works on hidden modules in Lua syntax (only with Lmod) - test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + test_modules_path = os.path.join(TEST_MODULES_DIR) if isinstance(self.modtool, Lmod): # make sure only the .lua module file is there, otherwise this test doesn't work as intended self.assertExists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua')) @@ -626,7 +625,7 @@ def test_prepend_module_path(self): self.assertEqual(modulepath, curr_module_paths()) # prepending path that is 'deeper down' in $MODULEPATH works, brings it back to front - test_mods_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_mods_dir = os.path.join(TEST_MODULES_DIR) self.assertTrue(any(os.path.samefile(test_mods_dir, p) for p in modulepath)) self.modtool.prepend_module_path(test_mods_dir) self.assertTrue(os.path.samefile(curr_module_paths()[0], test_mods_dir)) @@ -771,7 +770,7 @@ def check_get_software_libdir(expected, **additional_args): def test_wrong_modulepath(self): """Test whether modules tool can deal with a broken $MODULEPATH.""" - test_modules_path = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) + test_modules_path = os.path.realpath(os.path.join(TEST_MODULES_DIR)) modules_test_installpath = os.path.join(self.test_installpath, 'modules', 'all') os.environ['MODULEPATH'] = '/some/non-existing/path:/this/doesnt/exists/anywhere:%s' % test_modules_path init_config() @@ -785,8 +784,7 @@ def test_wrong_modulepath(self): def test_modulefile_path(self): """Test modulefile_path method""" - test_dir = os.path.abspath(os.path.dirname(__file__)) - gcc_mod_file = os.path.join(test_dir, 'modules', 'GCC', '6.4.0-2.28') + gcc_mod_file = os.path.join(TEST_MODULES_DIR, 'GCC', '6.4.0-2.28') modtool = modules_tool() res = modtool.modulefile_path('GCC/6.4.0-2.28') @@ -794,7 +792,7 @@ def test_modulefile_path(self): if isinstance(self.modtool, Lmod): res = modtool.modulefile_path('bzip2/.1.0.6') - self.assertTrue(os.path.samefile(res, os.path.join(test_dir, 'modules', 'bzip2', '.1.0.6.lua'))) + self.assertTrue(os.path.samefile(res, os.path.join(TEST_MODULES_DIR, 'bzip2', '.1.0.6.lua'))) res = modtool.modulefile_path('bzip2/.1.0.6', strip_ext=True) self.assertTrue(res.endswith('test/framework/modules/bzip2/.1.0.6')) @@ -814,11 +812,10 @@ def test_path_to_top_of_module_tree(self): def test_path_to_top_of_module_tree_hierarchical_mns(self): """Test function to determine path to top of the module tree for a hierarchical module naming scheme.""" - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -853,7 +850,7 @@ def test_path_to_top_of_module_tree_lua(self): """Test path_to_top_of_module_tree function on modules in Lua syntax.""" if isinstance(self.modtool, Lmod): orig_modulepath = os.environ.get('MODULEPATH') - self.modtool.unuse(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) + self.modtool.unuse(os.path.join(TEST_MODULES_DIR)) curr_modulepath = os.environ.get('MODULEPATH') error_msg = "Incorrect $MODULEPATH value after unuse: %s (orig: %s)" % (curr_modulepath, orig_modulepath) self.assertEqual(curr_modulepath, None, error_msg) @@ -1024,11 +1021,10 @@ def test_path_to_top_of_module_tree_categorized_hmns(self): scheme. """ - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, - 'robot_path': [ecs_dir], + 'robot_path': [TEST_ECS_DIR], 'valid_stops': all_stops, 'validate': False, } @@ -1063,7 +1059,7 @@ def test_path_to_top_of_module_tree_categorized_hmns(self): def test_modules_tool_stateless(self): """Check whether ModulesTool instance is stateless between runs.""" - test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_modules_path = os.path.join(TEST_MODULES_DIR) # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load copy_dir(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) @@ -1159,7 +1155,7 @@ def test_module_caches(self): # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH # invalidate_module_caches_for should be able to deal with this - test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + test_mods_path = os.path.join(TEST_MODULES_DIR) mods_symlink = os.path.join(self.test_prefix, 'modules_symlink') os.symlink(test_mods_path, mods_symlink) self.modtool.use(mods_symlink) @@ -1588,11 +1584,8 @@ def test_get_setenv_value_from_modulefile(self): init_config(build_options={'generate_devel_module': True}) - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(eb_path)) + write_file(test_ec, TOY_EC_TXT) write_file(test_ec, "\nmodextravars = {'FOO': 'value with spaces'}", append=True) toy_eb = EasyBlock(EasyConfig(test_ec)) diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 6c9cefd95d..55ba615075 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -41,6 +41,7 @@ from easybuild.tools.environment import join_path_var from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import EnvironmentModules, Lmod +from test.framework import TEST_MODULES_DIR from test.framework.utilities import init_config @@ -166,7 +167,7 @@ def test_lmod_specific(self): os.environ['PATH'] = join_path_var(new_paths) # make sure $MODULEPATH contains path that provides some modules - os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + os.environ['MODULEPATH'] = os.path.abspath(TEST_MODULES_DIR) # initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD fake_path = os.path.join(self.test_installpath, 'lmod') diff --git a/test/framework/options.py b/test/framework/options.py index 8c9c037ef0..1d02bd49a8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -66,6 +66,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type from easybuild.tools.version import VERSION +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config from test.framework.github import ignore_rate_limit_in_pr @@ -250,7 +251,7 @@ def test_force(self): """Test forcing installation even if the module is already available.""" # use GCC-4.6.3.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb') + eb_file = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') # check log message without --force args = [ @@ -279,12 +280,10 @@ def test_force(self): def test_skip(self): """Test skipping installation of module (--skip, -k).""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # check log message with --skip for existing module args = [ - toy_ec, + TOY_EC, '--force', '--debug', ] @@ -305,7 +304,7 @@ def test_skip(self): # check log message with --skip for non-existing module args = [ - toy_ec, + TOY_EC, '--try-software-version=1.2.3.4.5.6.7.8.9', '--try-amend=sources=toy-0.0.tar.gz,toy-0.0.tar.gz', # hackish, but fine '--force', @@ -328,7 +327,7 @@ def test_skip(self): # make sure that sanity check is *NOT* skipped under --skip test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT regex = re.compile(r"sanity_check_paths = \{(.|\n)*\}", re.M) test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt) write_file(test_ec, test_ec_txt) @@ -342,11 +341,9 @@ def test_skip(self): def test_module_only_param(self): """check use of module_only parameter""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\nmodule_only=True\n" test_ec_txt += "\nskipsteps = ['sanitycheck']\n" # Software does not exist, so sanity check would fail write_file(test_ec, test_ec_txt) @@ -369,12 +366,10 @@ def test_module_only_param(self): def test_skipsteps(self): """Test skipping of steps using skipsteps.""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # make sure that sanity check is *NOT* skipped test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT regex = re.compile(r"sanity_check_paths = \{(.|\n)*\}", re.M) test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt) write_file(test_ec, test_ec_txt) @@ -409,8 +404,7 @@ def test_skipsteps(self): def test_skip_test_step(self): """Test skipping testing the build (--skip-test-step).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') # check log message without --skip-test-step args = [ @@ -443,9 +437,8 @@ def test_skip_test_step(self): def test_ignore_test_failure(self): """Test ignore failing tests (--ignore-test-failure).""" - topdir = os.path.abspath(os.path.dirname(__file__)) # This EC uses a `runtest` command which does not exist and hence will make the test step fail - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb') args = [toy_ec, '--ignore-test-failure', '--force'] @@ -466,10 +459,8 @@ def test_ignore_test_failure(self): def test_skip_sanity_check(self): """Test skipping of sanity check step (--skip-sanity-check).""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec) + "\nsanity_check_commands = ['this_will_fail']") + write_file(test_ec, TOY_EC_TXT + "\nsanity_check_commands = ['this_will_fail']") args = [test_ec, '--rebuild'] err_msg = "Sanity check failed" @@ -491,7 +482,7 @@ def test_job(self): """Test submitting build as a job.""" # use gzip-1.4.eb easyconfig file that comes with the tests - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR eb_file = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb') def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb_file='gzip-1.4.eb'): @@ -535,7 +526,7 @@ def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb # check if libtoy dep uses --try-toolchain but gzip does not (easyconfig exists already) eb_file = os.path.join(self.test_buildpath, 'toy-0.0-with-deps.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), eb_file) + copy_file(TOY_EC, eb_file) write_file(eb_file, "dependencies = [('libtoy', '0.0'), ('gzip', '1.4')]\n", append=True) try_opts = " --try-toolchain='GCC,4.9.3-2.26'" tweaked_eb_file = "toy-0.0-GCC-4.9.3-2.26.eb" @@ -576,8 +567,7 @@ def test_zzz_logtostdout(self): error_msg = "Log messages are printed to stdout when %s is used (stdout: %s)" % (stdout_arg, stdout) self.assertTrue(len(stdout) > 100, error_msg) - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ecfile = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ecfile = TOY_EC self.logfile = None with self.mocked_stdout_stderr(mock_stderr=False): @@ -945,8 +935,7 @@ def test_avail_cfgfile_constants(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + copy_dir(TEST_ECS_DIR, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -1084,15 +1073,9 @@ def test_000_list_easyblocks(self): def test_search(self): """Test searching for easyconfigs.""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - - # simple search - args = [ - '--search=gzip', - '--robot=%s' % test_easyconfigs_dir, - ] + args = ['--robot=%s' % TEST_ECS_DIR] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search=gzip'], testing=False) txt = self.get_stdout() for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]: @@ -1100,12 +1083,8 @@ def test_search(self): self.assertRegex(txt, regex) # search w/ regex - args = [ - '--search=^gcc.*2.eb', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search=^gcc.*2.eb'], testing=False) txt = self.get_stdout() for ec in ['GCC-4.8.2.eb', 'GCC-4.9.2.eb']: @@ -1122,12 +1101,8 @@ def test_search(self): ] # test --search-filename - args = [ - '--search-filename=^gcc', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search-filename=^gcc'], testing=False) txt = self.get_stdout() for ec in gcc_ecs: @@ -1135,13 +1110,8 @@ def test_search(self): self.assertRegex(txt, regex) # test --search-filename --terse - args = [ - '--search-filename=^gcc', - '--terse', - '--robot=%s' % test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False) + self.eb_main(args + ['--search-filename=^gcc', '--terse'], testing=False) txt = self.get_stdout() for ec in gcc_ecs: @@ -1150,14 +1120,8 @@ def test_search(self): # also test --search-short/-S for search_arg in ['-S', '--search-short']: - args = [ - search_arg, - '^toy-0.0', - '-r', - test_easyconfigs_dir, - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, raise_error=True, verbose=True, testing=False) + self.eb_main(args + [search_arg, '^toy-0.0'], raise_error=True, verbose=True, testing=False) txt = self.get_stdout() self.assertRegex(txt, re.compile(r'^CFGS\d+=', re.M)) @@ -1165,13 +1129,8 @@ def test_search(self): self.assertRegex(txt, r" \* \$CFGS\d+/*%s" % ec) # combining --search with --try-* should not cause trouble; --try-* should just be ignored - args = [ - '--search=^gcc', - '--robot-paths=%s' % test_easyconfigs_dir, - '--try-toolchain-version=1.2.3', - ] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, testing=False, raise_error=True) + self.eb_main(args + ['--search=^gcc', '--try-toolchain-version=1.2.3'], testing=False, raise_error=True) txt = self.get_stdout() self.assertIn('GCC-4.9.2', txt) @@ -1180,9 +1139,8 @@ def test_search(self): # characters like ^, . or * are not touched, since these can be used as regex characters in queries for opt in ['--search', '-S', '--search-short']: for pattern in ['netCDF-C++', 'foo|bar', '^foo', 'foo.*bar']: - args = [opt, pattern, '--robot', test_easyconfigs_dir] with self.mocked_stdout_stderr(mock_stderr=False): - self.eb_main(args, raise_error=True, verbose=True, testing=False) + self.eb_main(args + [opt, pattern], raise_error=True, verbose=True, testing=False) stdout = self.get_stdout() # there shouldn't be any hits for any of these queries, so empty output... self.assertEqual(stdout.strip(), '') @@ -1192,14 +1150,14 @@ def test_search(self): # a proper error is produced in that case (as opposed to a crash) for opt in ['--search', '-S', '--search-short']: for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']: - args = [opt, pattern, '--robot', test_easyconfigs_dir] with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True) + self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args + [opt, pattern], + raise_error=True) # test searching for non-existing easyconfig file (should produce non-zero exit code) # 4 corresponds with MISSING_EASYCONFIG in EasyBuildExit (see easybuild/tools/build_log.py) - args = ['--search', 'nosuchsoftware-1.2.3.4.5'] - self.assertErrorRegex(SystemExit, 'MISSING_EASYCONFIG|4', self.eb_main, args, + self.assertErrorRegex(SystemExit, 'MISSING_EASYCONFIG|4', self.eb_main, + args + ['--search', 'nosuchsoftware-1.2.3.4.5'], testing=False, raise_error=True, raise_systemexit=True) def test_ignore_index(self): @@ -1207,9 +1165,7 @@ def test_ignore_index(self): Test use of --ignore-index. """ - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') - copy_file(toy_ec, self.test_prefix) + copy_file(TOY_EC, self.test_prefix) toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb'] @@ -1303,11 +1259,8 @@ def mocked_main(self, args, **kwargs): def test_copy_ec(self): """Test --copy-ec.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') - - toy_ec_txt = read_file(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')) - bzip2_ec_txt = read_file(os.path.join(test_easyconfigs_dir, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) + toy_ec_txt = TOY_EC_TXT + bzip2_ec_txt = read_file(os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1732,8 +1685,7 @@ def test_dry_run_short(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + copy_dir(TEST_ECS_DIR, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -1779,7 +1731,7 @@ def test_try_robot_force(self): os.close(fd) # use toy-0.0.eb easyconfig file that comes with the tests - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR eb1 = os.path.join(test_ecs, 'f', 'FFTW', 'FFTW-3.3.7-gompi-2018a.eb') eb2 = os.path.join(test_ecs, 's', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb') @@ -1815,7 +1767,7 @@ def test_try_robot_force(self): def test_try_toolchain_mapping(self): """Test mapping of subtoolchains with --try-toolchain.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') args = [ @@ -2043,7 +1995,7 @@ def test_github_from_pr(self): '--from-pr=22227', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -2072,7 +2024,7 @@ def test_github_from_pr(self): '--from-pr=22227,19834', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -2112,7 +2064,7 @@ def test_github_from_pr_token_log(self): '--dry-run', '--debug', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % TEST_DIR / 'easyconfigs', '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user ] try: @@ -2137,7 +2089,7 @@ def test_github_from_pr_listed_ecs(self): os.close(fd) # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR ecstmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(ecstmpdir, 'easybuild'), parents=True) copy_dir(test_ecs_path, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) @@ -2407,7 +2359,7 @@ def test_header_footer(self): write_file(modules_header, modules_header_txt) # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC # check log message with --skip for existing module args = [ @@ -2441,7 +2393,7 @@ def test_recursive_module_unload(self): """Test generating recursively unloading modules.""" # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + eb_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] @@ -2476,7 +2428,7 @@ def test_tmpdir(self): tmpdir = tempfile.mkdtemp() # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC # check log message with --skip for existing module args = [ @@ -2647,8 +2599,7 @@ def test_allow_modules_tool_mismatch(self): # trigger that main() creates new instance of ModulesTool self.modtool = None - topdir = os.path.abspath(os.path.dirname(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC # keep track of original module definition so we can restore it orig_module = os.environ.get('module', None) @@ -2702,15 +2653,13 @@ def test_allow_modules_tool_mismatch(self): def test_try(self): """Test whether --try options are taken into account.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) - args = [ tweaked_toy_ec, '--dry-run', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, ] test_cases = [ @@ -2764,7 +2713,7 @@ def test_try(self): # Try changing only name or version of toolchain args.pop(0) # Remove EC filename foss_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-foss-2018a.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) + copy_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) write_file(foss_toy_ec, "toolchain['name'] = 'foss'", append=True) test_cases = [ @@ -2779,15 +2728,14 @@ def test_try(self): def test_try_with_copy(self): """Test whether --try options are taken into account.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, '--dry-run', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--copy-ec', ] copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') @@ -2811,9 +2759,7 @@ def test_try_with_copy(self): def test_software_version_ordering(self): """Test whether software versions are correctly ordered when using --software.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - gcc_ec = os.path.join(ecs_path, 'g', 'GCC', 'GCC-4.9.2.eb') + gcc_ec = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.2.eb') test_gcc_ec = os.path.join(self.test_prefix, 'GCC-4.10.1.eb') test_gcc_txt = read_file(gcc_ec).replace("version = '4.9.2'", "version = '4.10.1'") @@ -2823,7 +2769,7 @@ def test_software_version_ordering(self): args = [ '--software=GCC,4.10.1', '--dry-run', - '--robot=%s:%s' % (ecs_path, self.test_prefix), + '--robot=%s:%s' % (TEST_ECS_DIR, self.test_prefix), ] with self.mocked_stdout_stderr(): out = self.eb_main(['--software=GCC,4.10.1'] + args[1:], raise_error=True) @@ -2833,17 +2779,16 @@ def test_software_version_ordering(self): def test_recursive_try(self): """Test whether recursive --try-X works.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + copy_file(TOY_EC, tweaked_toy_ec) write_file(tweaked_toy_ec, "dependencies = [('gzip', '1.4')]\n", append=True) # add fictious dependency - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') + sourcepath = os.path.join(TEST_DIR, 'sandbox', 'sources') args = [ tweaked_toy_ec, '--sourcepath=%s' % sourcepath, '--try-toolchain=gompi,2018a', - '--robot=%s' % ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--ignore-osdeps', '--dry-run', ] @@ -2906,11 +2851,10 @@ def test_recursive_try(self): def test_cleanup_builddir(self): """Test cleaning up of build dir and --disable-cleanup-builddir.""" - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toy_buildpath = os.path.join(self.test_buildpath, 'toy', '0.0', 'system-system') args = [ - toy_ec, + TOY_EC, '--force', ] with self.mocked_stdout_stderr(): @@ -2927,7 +2871,7 @@ def test_cleanup_builddir(self): # make sure build dir stays in case of failed build args = [ - toy_ec, + TOY_EC, '--force', '--try-amend=prebuildopts=nosuchcommand &&', ] @@ -2937,12 +2881,11 @@ def test_cleanup_builddir(self): def test_filter_deps(self): """Test use of --filter-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + os.environ['MODULEPATH'] = str(TEST_MODULES_DIR) args = [ ec_file, - '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--robot=%s' % TEST_ECS_DIR, '--dry-run', ] with self.mocked_stdout_stderr(): @@ -3051,7 +2994,7 @@ def test_filter_deps(self): # This easyconfig contains a dependency of CMake for which no easyconfig exists. It should still # succeed when called with --filter-deps=CMake=:2.8.10] write_file(self.logfile, '') - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' with self.mocked_stdout_stderr(): @@ -3062,7 +3005,7 @@ def test_filter_deps(self): # The test below fails without PR 2983 write_file(self.logfile, '') - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' with self.mocked_stdout_stderr(): @@ -3071,9 +3014,9 @@ def test_filter_deps(self): def test_hide_deps(self): """Test use of --hide-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + test_dir = TEST_DIR + ec_file = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') + os.environ['MODULEPATH'] = str(TEST_MODULES_DIR) args = [ ec_file, '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), @@ -3108,8 +3051,7 @@ def test_hide_deps(self): def test_hide_toolchains(self): """Test use of --hide-toolchains.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') args = [ ec_file, '--dry-run', @@ -3192,8 +3134,7 @@ def test_parse_http_header_fields_urlpat(self): def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" tmpdir = tempfile.mkdtemp() - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') gzip_ec_txt = read_file(gzip_ec) regex = re.compile('^source_urls = .*', re.M) test_ec_txt = regex.sub("source_urls = ['https://sources.easybuild.io/g/gzip']", gzip_ec_txt) @@ -3311,7 +3252,7 @@ def test_test_report_env_filter(self): def toy(extra_args=None): """Build & install toy, return contents of test report.""" - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC args = [ eb_file, '--force', @@ -3363,7 +3304,7 @@ def test_robot(self): # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp os.environ['EASYBUILD_ROBOT_PATHS'] = self.test_prefix - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR # includes 'toy/.0.0-deps' as a dependency eb_file = os.path.join(test_ecs_path, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') @@ -3689,7 +3630,7 @@ def test_xxx_include_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - test_easyblocks = os.path.dirname(os.path.abspath(__file__)) + test_easyblocks = TEST_DIR path_pattern = os.path.join(test_easyblocks, 'sandbox', 'easybuild', 'easyblocks', 'f', 'foo.py') foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertRegex(logtxt, foo_regex) @@ -3704,7 +3645,7 @@ def test_xxx_include_easyblocks(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -3826,7 +3767,7 @@ def test_xxx_include_generic_easyblocks(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -3970,7 +3911,7 @@ def test_github_xxx_include_easyblocks_from_pr(self): # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic, # to avoid that easyblocks picked up from other places cause trouble - testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): for path in sys.modules[pkg].__path__[:]: if testdir_sandbox not in path: @@ -4039,19 +3980,17 @@ def test_include_module_naming_schemes(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4101,8 +4040,7 @@ def test_use_included_module_naming_scheme(self): ]) write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt) - topdir = os.path.abspath(os.path.dirname(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb_file = TOY_EC args = [ '--unittest-file=%s' % self.logfile, '--module-naming-scheme=AnotherTestIncludedMNS', @@ -4129,19 +4067,17 @@ def test_include_toolchains(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4189,19 +4125,17 @@ def test_include_job_backends(self): # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH self.restore_env_path_pythonpath() - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # try and make sure 'eb' is available via $PATH if it isn't yet path = self.env_path if which('eb') is None: - path = '%s:%s' % (topdir, path) + path = '%s:%s' % (REPO_ROOT, path) # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) if res.exit_code != 0: - pythonpath = '%s:%s' % (topdir, pythonpath) + pythonpath = '%s:%s' % (REPO_ROOT, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -4237,9 +4171,9 @@ def test_include_job_backends(self): def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" - topdir = os.path.dirname(os.path.abspath(__file__)) + args = [ - os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), + TOY_EC, '--dry-run', '--try-software-version=1.0', # so we get a tweaked easyconfig ] @@ -4272,7 +4206,7 @@ def test_github_preview_pr(self): print("Skipping test_preview_pr, no GitHub token available?") return - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs_path = TEST_ECS_DIR eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') args = [ '--color=never', @@ -4433,7 +4367,7 @@ def test_minimal_toolchains(self): def test_extended_dry_run(self): """Test use of --extended-dry-run/-x.""" - ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC args = [ ec_file, '--debug', @@ -4502,8 +4436,8 @@ def test_last_log(self): def test_fixed_installdir_naming_scheme(self): """Test use of --fixed-installdir-naming-scheme.""" # by default, name of install dir match module naming scheme used - topdir = os.path.abspath(os.path.dirname(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + eb_file = TOY_EC app = EasyBlock(EasyConfig(eb_file)) app.gen_installdir() self.assertTrue(app.installdir.endswith('software/toy/0.0')) @@ -4543,16 +4477,10 @@ def test_new_branch_github(self): print("Skipping test_create_branch_github, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - - # test easyconfigs - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - args = [ '--new-branch-github', '--github-user=%s' % GITHUB_TEST_ACCOUNT, - toy_ec, + TOY_EC, '-D', ] txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) @@ -4566,7 +4494,7 @@ def test_new_branch_github(self): self.assert_multi_regex(regexs, txt) # test easyblocks - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') toy_eb = os.path.join(test_ebs, 't', 'toy.py') args = [ @@ -4587,7 +4515,7 @@ def test_new_branch_github(self): self.assert_multi_regex(regexs, txt) # test framework with tweaked copy of test_module_naming_scheme.py - test_mns_py = os.path.join(topdir, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', + test_mns_py = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', 'test_module_naming_scheme.py') target_dir = os.path.join(self.test_prefix, 'easybuild-framework', 'test', 'framework', 'sandbox', 'easybuild', 'tools', 'module_naming_scheme') @@ -4659,14 +4587,10 @@ def test_update_branch_github(self): print("Skipping test_update_branch_github, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - args = [ '--update-branch-github=develop', '--github-user=boegel', # used to determine account to grab branch from (no GitHub token needed) - toy_ec, + TOY_EC, '--pr-commit-msg="this is just a test"', '--force', # force required because we're using --pr-commit-msg when only adding new easyconfigs '-D', @@ -4690,11 +4614,11 @@ def test_github_new_update_pr(self): return # copy toy test easyconfig - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') + + test_ecs = TEST_ECS_DIR toy_ec = os.path.join(self.test_prefix, 'toy.eb') toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' - toy_patch = os.path.join(topdir, 'sandbox', 'sources', 'toy', toy_patch_fn) + toy_patch = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', toy_patch_fn) # purposely picked one with non-default toolchain/versionsuffix copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), toy_ec) @@ -4917,10 +4841,8 @@ def test_github_new_pr_warning_missing_patch(self): print("Skipping test_new_pr_warning_missing_patch, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), test_ec) + copy_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), test_ec) patches_regex = re.compile(r'^patches = .*', re.M) test_ec_txt = read_file(test_ec) @@ -5017,9 +4939,9 @@ def test_github_new_pr_python(self): return # copy toy test easyconfig - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR toy_ec = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), toy_ec) + copy_file(TOY_EC, toy_ec) # modify file to include Python dependency toy_ec_txt = read_file(toy_ec) @@ -5138,8 +5060,7 @@ def test_github_new_pr_easyblock(self): print("Skipping test_new_pr_easyblock, no GitHub token available?") return - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertExists(toy_eb) args = [ @@ -5321,7 +5242,7 @@ def test_show_config(self): default_prefix = os.path.join(os.environ['HOME'], '.local', 'easybuild') - test_dir = os.path.dirname(os.path.abspath(__file__)) + test_dir = TEST_DIR expected_lines = [ r"#", r"# Current EasyBuild configuration", @@ -5334,7 +5255,7 @@ def test_show_config(self): r"ignoreconfigfiles\s* \(E\) = %s" % ', '.join(os.environ['EASYBUILD_IGNORECONFIGFILES'].split(',')), r"installpath\s* \(E\) = " + os.path.join(self.test_prefix, 'tmp.*'), r"repositorypath\s* \(D\) = " + os.path.join(default_prefix, 'ebfiles_repo'), - r"robot-paths\s* \(E\) = " + os.path.join(test_dir, 'easyconfigs', 'test_ecs'), + r"robot-paths\s* \(E\) = " + os.path.join(TEST_ECS_DIR), r"rpath\s* \(D\) = " + ('False' if get_os_type() == DARWIN else 'True'), r"sourcepath\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'sources'), r"sourcepath-data\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'data_sources'), @@ -5617,8 +5538,7 @@ def test_fetch_all(self): mkdir(lock_path, parents=True) # copy toy-0.0.eb test easyconfig, tweak version to something that no source can be obtained for - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT test_ec_txt = toy_ec_txt.replace("version = '0.0'", "version = '1.2.3.4.5.6'") test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -5842,12 +5762,11 @@ def test_list_software(self): # copy selected test easyconfigs for testing --list-*software options with; # full test is a nuisance, because all dependencies must be available and toolchains like intel must have # all expected components when testing with HierarchicalMNS (which the test easyconfigs don't always have) - topdir = os.path.dirname(os.path.abspath(__file__)) - cray_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb') - gcc_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb') - gzip_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') - gzip_system_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb') + cray_ec = os.path.join(TEST_ECS_DIR, 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb') + gcc_ec = os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.6.3.eb') + gzip_ec = os.path.join(TEST_DIR, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + gzip_system_ec = os.path.join(TEST_DIR, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb') test_ecs = os.path.join(self.test_prefix, 'test_ecs') for ec in [cray_ec, gcc_ec, gzip_ec, gzip_system_ec]: @@ -6023,7 +5942,7 @@ def test_check_contrib_style(self): # copy toy-0.0.eb test easyconfig, fiddle with it to make style check fail toy = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), toy) + copy_file(TOY_EC, toy) toytxt = read_file(toy) # introduce whitespace issues @@ -6078,7 +5997,7 @@ def test_check_contrib_non_style(self): # --check-contrib passes if None values are used as checksum, but produces warning toy = os.path.join(self.test_prefix, 'toy.eb') - copy_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), toy) + copy_file(TOY_EC, toy) toytxt = read_file(toy) toytxt = toytxt + '\n'.join([ 'checksums = [', @@ -6118,13 +6037,11 @@ def test_allow_use_as_root(self): def test_verify_easyconfig_filenames(self): """Test --verify-easyconfig-filename""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - toy_ec = os.path.join(test_easyconfigs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) args = [ test_ec, @@ -6151,7 +6068,7 @@ def test_verify_easyconfig_filenames(self): write_file(self.logfile, '') - args[0] = toy_ec + args[0] = TOY_EC with self.mocked_stdout_stderr(): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) @@ -6159,8 +6076,8 @@ def test_verify_easyconfig_filenames(self): def test_set_default_module(self): """Test use of --set-default-module""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') with self.mocked_stdout_stderr(): self.eb_main([toy_ec, '--set-default-module'], do_build=True, raise_error=True) @@ -6264,8 +6181,8 @@ def test_set_default_module_robot(self): def test_inject_checksums(self): """Test for --inject-checksums""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # checksums are injected in existing easyconfig, so test with a copy test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -6401,8 +6318,7 @@ def test_inject_checksums(self): remove_file(ec_backups[0]) # also test injecting of MD5 checksums into easyconfig that doesn't include checksums already - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT # get rid of existing checksums regex = re.compile(r'^checksums(?:.|\n)*?\]\s*$', re.M) @@ -6444,7 +6360,7 @@ def test_inject_checksums(self): self.assertEqual(ec['checksums'], checksums) # check whether empty list of checksums is stripped out by --inject-checksums - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT regex = re.compile(r'^checksums(?:.|\n)*?\]\s*$', re.M) toy_ec_txt = regex.sub('', toy_ec_txt) @@ -6505,7 +6421,7 @@ def test_inject_checksums(self): self.assertEqual(ext_opts['checksums'], expected_checksums) # Also works for cargo crates - cargo_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-cargo.eb') + cargo_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-cargo.eb') copy_file(cargo_ec, test_ec) stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True) self.assertIn("injecting sha256 checksums in", stdout) @@ -6567,10 +6483,8 @@ def test_inject_checksums(self): def test_inject_checksums_to_json(self): """Test --inject-checksums-to-json.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - copy_file(toy_ec, test_ec) + copy_file(TOY_EC, test_ec) test_ec_txt = read_file(test_ec) args = [test_ec, '--inject-checksums-to-json'] @@ -6596,15 +6510,14 @@ def test_inject_checksums_to_json(self): def test_force_download(self): """Test --force-download""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_srcdir = os.path.join(topdir, 'sandbox', 'sources', 'toy') - copy_file(toy_ec, self.test_prefix) + toy_srcdir = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') + + copy_file(TOY_EC, self.test_prefix) toy_tar = 'toy-0.0.tar.gz' copy_file(os.path.join(toy_srcdir, toy_tar), os.path.join(self.test_prefix, 't', 'toy', toy_tar)) - toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) + toy_ec = os.path.join(self.test_prefix, os.path.basename(TOY_EC)) write_file(toy_ec, "\nsource_urls = ['file://%s']" % toy_srcdir, append=True) args = [ @@ -6624,8 +6537,8 @@ def test_force_download(self): def test_enforce_checksums(self): """Test effect of --enforce-checksums""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') # wipe $EASYBUILD_ROBOT_PATHS to avoid that checksums.json for toy is found in test_ecs @@ -6724,9 +6637,6 @@ def test_check_eb_deps(self): def test_tmp_logdir(self): """Test use of --tmp-logdir.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - # purposely use a non-existing directory as log directory tmp_logdir = os.path.join(self.test_prefix, 'tmp-logs') self.assertNotExists(tmp_logdir) @@ -6736,7 +6646,7 @@ def test_tmp_logdir(self): # check log message with --skip for existing module args = [ - toy_ec, + TOY_EC, '--force', '--debug', '--tmp-logdir=%s' % tmp_logdir, @@ -6752,11 +6662,9 @@ def test_tmp_logdir(self): def test_sanity_check_only(self): """Test use of --sanity-check-only.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.ec') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", @@ -6865,7 +6773,7 @@ def test_sanity_check_only(self): self.eb_main(args, do_build=True, raise_error=True) # also check when using easyblock that enables build_in_installdir in its constructor - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + test_ebs = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') toy_eb = os.path.join(test_ebs, 't', 'toy.py') toy_eb_txt = read_file(toy_eb) @@ -6899,11 +6807,9 @@ def test_sanity_check_only(self): def test_keep_going(self): """Test use of --keep-going.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nsources=["toy-0.0.tar.gz"]' write_file(test_ec, test_ec_txt + '\nversion="broken"\npreconfigopts = "false && "') test_ec2 = os.path.join(self.test_prefix, 'test2.eb') @@ -6936,12 +6842,10 @@ def test_keep_going(self): def test_skip_extensions(self): """Test use of --skip-extensions.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # add extension, which should be skipped test_ec = os.path.join(self.test_prefix, 'test.ec') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "exts_list = [", " ('barbar', '0.0', {", @@ -6970,9 +6874,6 @@ def test_skip_extensions(self): def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_mns = os.path.join(self.test_prefix, 'test_mns.py') test_mns_txt = '\n'.join([ "import vsc", @@ -6983,7 +6884,7 @@ def test_fake_vsc_include(self): write_file(test_mns, test_mns_txt) args = [ - toy_ec, + TOY_EC, '--dry-run', '--include-module-naming-schemes=%s' % test_mns, ] @@ -6996,22 +6897,19 @@ def test_fake_vsc_include(self): def test_installdir(self): """Check naming scheme of installation directory.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/toy/0.0')) # even with HierarchicalMNS the installation directory remains the same, # due to --fixed-installdir-naming-scheme being enabled by default args = ['--module-naming-scheme=HierarchicalMNS'] init_config(args=args) - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/toy/0.0')) # things change when --disable-fixed-installdir-naming-scheme is used init_config(args=args, build_options={'fixed_installdir_naming_scheme': False}) - eb = EasyBlock(EasyConfig(toy_ec)) + eb = EasyBlock(EasyConfig(TOY_EC)) self.assertTrue(eb.installdir.endswith('/software/Core/toy/0.0')) def test_cuda_compute_capabilities(self): @@ -7024,7 +6922,7 @@ def test_cuda_compute_capabilities(self): def test_create_index(self): """Test --create-index option.""" - test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + test_ecs = TEST_ECS_DIR remove_dir(self.test_prefix) copy_dir(test_ecs, self.test_prefix) @@ -7113,13 +7011,12 @@ def test_accept_eula_for(self): """Test --accept-eula-for configuration option.""" # use toy-0.0.eb easyconfig file that comes with the tests - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ "easyblock = 'EB_toy_eula'", '', - read_file(toy_ec), + TOY_EC_TXT, ]) write_file(test_ec, test_ec_txt) @@ -7239,8 +7136,8 @@ def test_config_repositorypath(self): # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') + + toy_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_nonexistent.yaml') args = ['--easystack', toy_easystack, '--experimental'] expected_err = "No such file or directory: '%s'" % toy_easystack with self.mocked_stdout_stderr(): @@ -7250,8 +7147,8 @@ def test_easystack_wrong_read(self): # expecting successful build def test_easystack_basic(self): """Test for --easystack -> success case""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + toy_easystack = os.path.join(TEST_DIR, 'easystacks', 'test_easystack_basic.yaml') args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run'] with self.mocked_stdout_stderr(): @@ -7334,12 +7231,11 @@ def test_easystack_easyconfigs_cache(self): Test for easystack file that specifies same easyconfig twice, but from a different location. """ - topdir = os.path.abspath(os.path.dirname(__file__)) - libtoy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'l', 'libtoy', 'libtoy-0.0.eb') - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + libtoy_ec = os.path.join(TEST_ECS_DIR, 'l', 'libtoy', 'libtoy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\ndependencies = [('libtoy', '0.0')]" write_file(test_ec, test_ec_txt) diff --git a/test/framework/package.py b/test/framework/package.py index eb9b7e53c8..7a1326b12a 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -33,6 +33,7 @@ import sys from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_ECS_DIR from unittest import TextTestRunner from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -181,9 +182,7 @@ def test_active_pns(self): os.environ['EASYBUILD_PACKAGE_NAMING_SCHEME'] = pns_type init_config(build_options={'silent': True}) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + test_ec = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') ec = EasyConfig(test_ec, validate=False) pns = ActivePNS() @@ -206,9 +205,7 @@ def test_package(self): } init_config(build_options=build_options) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False) mock_fpm(self.test_prefix) @@ -246,7 +243,7 @@ def test_package(self): res = no_logfiles_regex.search(pkgtxt) self.assertFalse(res, "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt)) - toy_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')) + toy_txt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')) replace_str = '''description = """Toy C program, 100% toy. Now with `backticks'\n''' replace_str += '''and newlines"""''' toy_txt = re.sub('description = .*', replace_str, toy_txt) diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 9022fd3953..acfcd45533 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -31,6 +31,7 @@ import re import stat import sys +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -38,7 +39,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, update_build_option -from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_dir, which, write_file +from easybuild.tools.filetools import adjust_permissions, mkdir, remove_dir, which, write_file from easybuild.tools.job import pbs_python from easybuild.tools.job.pbs_python import PbsPython from easybuild.tools.options import parse_options @@ -128,18 +129,16 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob - topdir = os.path.dirname(os.path.abspath(__file__)) - build_options = { 'external_modules_metadata': {}, - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) @@ -148,7 +147,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) @@ -182,8 +181,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): self.assertIn('GCC-4.6.3.eb', jobs[3].deps[1].script) # also test use of --pre-create-installdir - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) + ordered_ecs = resolve_dependencies(process_easyconfig(TOY_EC), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') @@ -242,27 +240,23 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) - topdir = os.path.dirname(os.path.abspath(__file__)) - build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - easyconfigs = process_easyconfig(ec_file) + easyconfigs = process_easyconfig(TOY_EC) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) - topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') - cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) + test_easyblocks_path = os.path.join(TEST_DIR, 'sandbox') + cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (REPO_ROOT, test_easyblocks_path) with self.mocked_stdout_stderr(): build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) @@ -275,10 +269,9 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): # also check what happens when a job fails (an error should be raised) test_ecfile = os.path.join(self.test_prefix, 'test.eb') - ectxt = read_file(ec_file) # use different version, for which no sources are available regex = re.compile('^version = .*', re.M) - ectxt = regex.sub("version = '1.2.3'", ectxt) + ectxt = regex.sub("version = '1.2.3'", TOY_EC_TXT) write_file(test_ecfile, ectxt) ecs = resolve_dependencies(process_easyconfig(test_ecfile), self.modtool) @@ -288,8 +281,6 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): def test_submit_jobs(self): """Test submit_jobs""" - test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = process_easyconfig(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')) args = [ '--debug', @@ -302,7 +293,7 @@ def test_submit_jobs(self): '--job-cores=3', ] eb_go = parse_options(args=args) - cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True) + cmd = submit_jobs(process_easyconfig(TOY_EC), eb_go.generate_cmd_line(), testing=True) # these patterns must be found regexs = [ @@ -330,7 +321,7 @@ def test_submit_jobs(self): # test again with custom EasyBuild command to use in jobs update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug") - cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True) + cmd = submit_jobs(process_easyconfig(TOY_EC), eb_go.generate_cmd_line(), testing=True) regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ") self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd)) @@ -348,13 +339,12 @@ def test_build_easyconfigs_in_parallel_slurm(self): os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') - foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') + test_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + foss_ec = os.path.join(TEST_ECS_DIR, 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, - 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, diff --git a/test/framework/repository.py b/test/framework/repository.py index a5e992235d..470ba3b787 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -32,6 +32,7 @@ import shutil import sys import tempfile +from test.framework import TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner, mock @@ -100,8 +101,7 @@ def test_gitrepo(self): if res.exit_code == 0: repo = GitRepository(os.path.join(tmpdir, 'testrepository.git')) repo.init() - toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - repo.add_easyconfig(toy_ec_file, 'test', '1.0', {}, None) + repo.add_easyconfig(TOY_EC, 'test', '1.0', {}, None) with mock.patch.dict(os.environ, {'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@test.org', 'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@test.org'}): repo.commit("toy/0.0") @@ -146,7 +146,6 @@ def test_init_repository(self): def test_add_easyconfig(self): """Test use of add_easyconfig method""" repo = init_repository('FileRepository', self.path) - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') def check_ec(path, expected_buildstats): """Check easyconfig at specified path""" @@ -157,12 +156,10 @@ def check_ec(path, expected_buildstats): ecdict = EasyConfigParser(path).get_config_dict() self.assertEqual(ecdict['buildstats'], expected_buildstats) - toy_eb_file = os.path.join(test_easyconfigs, 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23}, None) + path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23}, None) check_ec(path, [{'time': 1.23}]) - path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}]) + path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}]) check_ec(path, [{'time': 0.9, 'size': 2}, {'time': 1.23, 'size': 123}]) def tearDown(self): diff --git a/test/framework/robot.py b/test/framework/robot.py index 326e97cbe5..413ebae8c4 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -35,6 +35,7 @@ import tempfile from copy import deepcopy from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_ECS_DIR, TOY_EC, TOY_EC_TXT from unittest import TextTestRunner import easybuild.framework.easyconfig.easyconfig as ecec @@ -137,7 +138,7 @@ def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ self.install_mock_module() - base_easyconfig_dir = find_full_path(os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')) + base_easyconfig_dir = find_full_path(TEST_ECS_DIR) self.assertTrue(base_easyconfig_dir) easyconfig = { @@ -403,7 +404,6 @@ def test_resolve_dependencies_minimal(self): # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') self.install_mock_module() init_config(build_options={ @@ -411,7 +411,7 @@ def test_resolve_dependencies_minimal(self): 'minimal_toolchains': True, 'use_existing_modules': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) @@ -509,14 +509,14 @@ def test_resolve_dependencies_minimal(self): 'minimal_toolchains': True, 'add_system_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) - impi_txt = read_file(os.path.join(test_easyconfigs, 'i', 'impi', 'impi-5.1.2.150.eb')) + impi_txt = read_file(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb')) self.assertTrue(re.search("^toolchain = SYSTEM", impi_txt, re.M)) - gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb')) + gzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb')) self.assertTrue(re.search("^toolchain = SYSTEM", gzip_txt, re.M)) barec = os.path.join(self.test_prefix, 'bar-1.2.3-foss-2018a.eb') @@ -551,8 +551,7 @@ def test_resolve_dependencies_missing(self): self.install_mock_module() MockModule.avail_modules = [] - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - init_config(build_options={'robot_path': [test_easyconfigs, self.test_prefix]}) + init_config(build_options={'robot_path': [str(TEST_ECS_DIR), self.test_prefix]}) ec = { 'ec': { @@ -615,21 +614,19 @@ def test_det_easyconfig_paths(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) # copy hwloc easyconfig to h/hwloc subdir in robot search path, # to trigger bug fixed in det_easyconfig_paths (.extend rather than .append for '__archive'__ to ignore_subdirs) hwloc_ec = 'hwloc-1.11.8-GCC-6.4.0-2.28.eb' subdir_hwloc = os.path.join(self.test_prefix, 'h', 'hwloc') mkdir(subdir_hwloc, parents=True) - shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) - shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', hwloc_ec), subdir_hwloc) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertNotExists(test_ec) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + TOY_EC, test_ec, # relative path, should be resolved via robot search path hwloc_ec, '--dry-run', @@ -643,7 +640,7 @@ def test_det_easyconfig_paths(self): outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ - (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (TEST_ECS_DIR, 'toy/0.0'), # specified easyconfigs, available at given location (self.test_prefix, 'intel/2018a'), # dependency, found in robot search path (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path (self.test_prefix, 'hwloc/1.11.8-GCC-6.4.0-2.28'), # specified easyconfig, found in robot search path @@ -670,9 +667,9 @@ def test_det_easyconfig_paths(self): self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + TOY_EC, + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), '--dry-run', '--robot', '--tmpdir=%s' % self.test_prefix, @@ -691,10 +688,8 @@ def test_search_paths(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) self.assertNotExists(test_ec) args = [ @@ -715,11 +710,9 @@ def test_github_det_easyconfig_paths_from_commit(self): """Test det_easyconfig_paths function in combination with --from-commit.""" # note: --from-commit does not involve using GitHub API, so no GitHub token required - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - commit = '589282cf52609067616fc2a522f8e4b81f809cb7' args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path + TOY_EC, # absolute path 'toy-0.0-iter.eb', # relative path, available via robot search path # commit in which ReFrame-4.3.2.eb was added, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits @@ -727,7 +720,7 @@ def test_github_det_easyconfig_paths_from_commit(self): 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path '--dry-run', '--robot', - '--robot=%s' % test_ecs_path, + '--robot=%s' % TEST_ECS_DIR, '--unittest-file=%s' % self.logfile, '--tmpdir=%s' % self.test_prefix, ] @@ -760,11 +753,9 @@ def test_github_det_easyconfig_paths_from_pr(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - test_ec = 'toy-0.0-deps.eb' - shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) - shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', test_ec), self.test_prefix) + shutil.copy2(os.path.join(TEST_ECS_DIR, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertNotExists(test_ec) gompi_2018b_txt = '\n'.join([ @@ -779,7 +770,7 @@ def test_github_det_easyconfig_paths_from_pr(self): write_file(os.path.join(self.test_prefix, 'gompi-2018b-test.eb'), gompi_2018b_txt) args = [ - os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + TOY_EC, test_ec, # relative path, should be resolved via robot search path # PR for XCrySDen/1.6.2-foss-2024a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227 '--from-pr=22227', @@ -815,10 +806,9 @@ def test_github_det_easyconfig_paths_from_pr(self): def test_get_toolchain_hierarchy(self): """Test get_toolchain_hierarchy function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) fosscuda_hierarchy = get_toolchain_hierarchy({'name': 'fosscuda', 'version': '2018a'}) @@ -849,7 +839,7 @@ def test_get_toolchain_hierarchy(self): # test also --try-toolchain* case, where we want more detailed information init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() @@ -965,7 +955,7 @@ def test_get_toolchain_hierarchy(self): init_config(build_options={ 'add_system_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() @@ -997,7 +987,7 @@ def test_get_toolchain_hierarchy(self): build_options = { 'add_system_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), } init_config(build_options=build_options) @@ -1019,13 +1009,13 @@ def test_get_toolchain_hierarchy(self): # put faulty foss easyconfig in place to test error reporting broken_gompi = os.path.join(self.test_prefix, 'gompi-2018a.eb') - copy_file(os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb'), broken_gompi) + copy_file(os.path.join(TEST_ECS_DIR, 'g', 'gompi', 'gompi-2018a.eb'), broken_gompi) ectxt = read_file(broken_gompi) ectxt += "\ndependencies += [('GCC', '4.6.4')]" write_file(broken_gompi, ectxt) init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': [self.test_prefix, test_easyconfigs], + 'robot_path': [self.test_prefix, TEST_ECS_DIR], }) tc = {'name': 'gompi', 'version': '2018a'} error_msg = "Multiple versions of GCC found in dependencies of toolchain gompi: 4.6.4, 6.4.0-2.28" @@ -1116,11 +1106,9 @@ def test_tweak_robotpath(self): """Test that the robot correctly resolves the dependencies of tweaked easyconfigs. Tweaked easyconfigs take priority, but tweaked dependencies are only used on an as-needed basis""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # Create directories to store the tweaked easyconfigs tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) + robot_path = det_robot_path([TEST_ECS_DIR], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), @@ -1129,7 +1117,7 @@ def test_tweak_robotpath(self): }) # Parse the easyconfig that we want to tweak - untweaked_openmpi = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi, False)]) # Tweak the toolchain version of the easyconfig @@ -1149,7 +1137,7 @@ def test_tweak_robotpath(self): # Check it picks up the tweaked OpenMPI self.assertIn(tweaked_openmpi, specs) # Check it picks up the untweaked dependency of the tweaked OpenMPI - untweaked_hwloc = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb') + untweaked_hwloc = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb') self.assertIn(untweaked_hwloc, specs) # Check correctness of tweak_map (maps back to the original untweaked file, even for hwloc, where the # tweaked version is generated but not used) @@ -1159,8 +1147,7 @@ def test_tweak_robotpath(self): def test_robot_find_subtoolchain_for_dep(self): """Test robot_find_subtoolchain_for_dep.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - init_config(build_options={'robot_path': test_easyconfigs}) + init_config(build_options={'robot_path': TEST_ECS_DIR}) # # First test that it can do basic resolution @@ -1192,7 +1179,7 @@ def test_robot_find_subtoolchain_for_dep(self): # init_config(build_options={ 'add_system_to_minimal_toolchains': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) # specify alternative parent toolchain gompi_1410 = {'name': 'gompi', 'version': '2018a'} @@ -1244,7 +1231,7 @@ def test_robot_find_subtoolchain_for_dep(self): write_file(barec, barec_txt) # check without --minimal-toolchains - init_config(build_options={'robot_path': test_easyconfigs}) + init_config(build_options={'robot_path': TEST_ECS_DIR}) bar = EasyConfig(barec) expected_dep_versions = { @@ -1260,7 +1247,7 @@ def test_robot_find_subtoolchain_for_dep(self): # check with --minimal-toolchains enabled init_config(build_options={ 'minimal_toolchains': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) bar = EasyConfig(barec) @@ -1294,7 +1281,7 @@ def test_robot_find_subtoolchain_for_dep(self): init_config(build_options={ 'minimal_toolchains': True, 'use_existing_modules': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) # Check gompi is now being picked up @@ -1315,10 +1302,9 @@ def test_robot_find_subtoolchain_for_dep_ecs_vs_mods(self): Test behaviour of robot_find_subtoolchain_for_dep w.r.t. picking subtoolchains based on easyconfigs vs modules. """ - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # include both test easyconfig files and test directory in robot search path - build_options = {'robot_path': [test_easyconfigs, self.test_prefix]} + build_options = {'robot_path': [str(TEST_ECS_DIR), self.test_prefix]} init_config(build_options=build_options) test_mods_dir = os.path.join(self.test_prefix, 'modules') @@ -1399,17 +1385,16 @@ def test_robot_find_subtoolchain_for_dep_ecs_vs_mods(self): def test_check_conflicts(self): """Test check_conflicts function.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'force': True, 'retain_all_deps': True, - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'valid_module_classes': module_classes(), 'validate': False, }) - gzip_ec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') - gompi_ec = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') + gompi_ec = os.path.join(TEST_ECS_DIR, 'g', 'gompi', 'gompi-2018a.eb') non_conflict_ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)]) # no conflicts found, no output to stderr @@ -1436,7 +1421,7 @@ def test_check_conflicts(self): self.assertIn("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28", stderr) # Can also return the text - with self.mocked_stdout_stderr(mock_stdout=False) as mocked_stderr: + with self.mocked_stderr() as mocked_stderr: conflict_lst = check_conflicts(ecs, self.modtool, return_conflicts=True) self.assertEqual('\n'.join(conflict_lst), stderr.strip()) self.assertEqual(mocked_stderr.getvalue(), '') @@ -1446,8 +1431,8 @@ def test_check_conflicts(self): # direct conflict on software version ecs, _ = parse_easyconfigs([ - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-6.4.0-2.28.eb'), False), - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), + (os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-6.4.0-2.28.eb'), False), + (os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), ]) with self.mocked_stderr(): conflicts = check_conflicts(ecs, self.modtool) @@ -1458,8 +1443,8 @@ def test_check_conflicts(self): # indirect conflict on dependencies ecs, _ = parse_easyconfigs([ - (os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), False), - (os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), False), + (os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), False), + (os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), False), ]) with self.mocked_stderr(): conflicts = check_conflicts(ecs, self.modtool) @@ -1472,9 +1457,9 @@ def test_check_conflicts(self): self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found") # Conflict in build dependencies is fine - hwloc_txt = read_file(os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) - gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) - bzip_txt = read_file(os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) + hwloc_txt = read_file(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + gzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + bzip_txt = read_file(os.path.join(TEST_ECS_DIR, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) tc = re.search(r"toolchain *=.*", hwloc_txt)[0] bzip_txt += f"\n{tc}" gzip_txt += f"\n{tc}" @@ -1495,9 +1480,6 @@ def test_check_conflicts(self): def test_check_conflicts_wrapper_deps(self): """Test check_conflicts when dependency 'wrappers' are involved.""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - wrapper_ec_txt = '\n'.join([ "easyblock = 'ModuleRC'", "name = 'toy'", @@ -1510,7 +1492,7 @@ def test_check_conflicts_wrapper_deps(self): wrapper_ec = os.path.join(self.test_prefix, 'toy-0.eb') write_file(wrapper_ec, wrapper_ec_txt) - ecs, _ = parse_easyconfigs([(toy_ec, False), (wrapper_ec, False)]) + ecs, _ = parse_easyconfigs([(TOY_EC, False), (wrapper_ec, False)]) with self.mocked_stderr(): res = check_conflicts(ecs, self.modtool) stderr = self.get_stderr() @@ -1520,12 +1502,9 @@ def test_check_conflicts_wrapper_deps(self): def test_check_conflicts_multi_deps(self): """Test check_conflicts when multi_deps is used.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT tc_regex = re.compile(r'^toolchain = .*', re.M) test_ec_txt = tc_regex.sub("toolchain = SYSTEM", test_ec_txt) test_ec_txt += "\nmulti_deps = {'GCC': ['4.9.2', '7.3.0-2.30']}\n" @@ -1534,7 +1513,7 @@ def test_check_conflicts_multi_deps(self): write_file(test_ec, test_ec_txt) ecs, _ = parse_easyconfigs([(test_ec, False)]) - init_config(build_options={'robot_path': [test_ecs]}) + init_config(build_options={'robot_path': [TEST_ECS_DIR]}) # use of multi_deps should not result in false positives in check_conflicts self.assertFalse(check_conflicts(ecs, self.modtool)) @@ -1546,9 +1525,7 @@ def test_robot_archived_easyconfigs(self): self.allow_deprecated_behaviour() init_config(build_options={'silent': True}) - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - - gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-intel-2018a.eb') + gzip_ec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-intel-2018a.eb') gzip_ectxt = read_file(gzip_ec) test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1563,29 +1540,28 @@ def test_robot_archived_easyconfigs(self): # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs init_config(build_options={ 'consider_archived_easyconfigs': True, - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, }) res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) self.assertEqual([ec['full_mod_name'] for ec in res], ['intel/2012a', 'gzip/1.5-intel-2012a']) - expected = os.path.join(test_ecs, '__archive__', 'i', 'intel', 'intel-2012a.eb') + expected = os.path.join(TEST_ECS_DIR, '__archive__', 'i', 'intel', 'intel-2012a.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected)) def test_search_easyconfigs(self): """Test search_easyconfigs function.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_ecs], + 'robot_path': [TEST_ECS_DIR], 'search_paths': [self.test_prefix], }) # copy some files to search_paths location - copy_file(os.path.join(test_ecs, 'b', 'binutils', 'binutils-2.25-GCCcore-4.9.3.eb'), self.test_prefix) - copy_file(os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), self.test_prefix) + copy_file(os.path.join(TEST_ECS_DIR, 'b', 'binutils', 'binutils-2.25-GCCcore-4.9.3.eb'), self.test_prefix) + copy_file(os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), self.test_prefix) paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', consider_extra_paths=False, print_result=False) - ref_paths = [os.path.join(test_ecs, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb']] + ref_paths = [os.path.join(TEST_ECS_DIR, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb']] self.assertEqual(len(paths), 2) self.assertEqual(paths, ref_paths) @@ -1598,17 +1574,17 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), + os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), + os.path.join(TEST_ECS_DIR, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') ] self.assertEqual(paths, ref_paths) # now do a case sensitive search paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False, case_sensitive=True) - ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] + ref_paths = [os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] self.assertEqual(paths, ref_paths) # test use of filename_only @@ -1631,7 +1607,7 @@ def test_search_easyconfigs(self): if filename_only: path = ec_fn else: - path = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs', 'b', 'binutils', ec_fn) + path = os.path.join(TEST_ECS_DIR, 'b', 'binutils', ec_fn) pattern.append(r"^ \* .*%s$" % path) regex = re.compile('\n'.join(pattern), re.M) diff --git a/test/framework/style.py b/test/framework/style.py index b8e74ca777..4022a30ffe 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -29,8 +29,8 @@ """ import glob -import os import sys +from test.framework import TEST_ECS_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -53,8 +53,7 @@ def test_style_conformance(self): return # all available easyconfig files - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - specs = glob.glob('%s/*.eb' % test_easyconfigs_path) + specs = glob.glob('%s/*.eb' % TEST_ECS_DIR) specs = sorted(specs) result = check_easyconfigs_style(specs) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index e1206e08a4..f87d7a0af7 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -37,6 +37,7 @@ import textwrap from itertools import product from unittest import TextTestRunner +from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config import easybuild.tools.modules as modules @@ -92,8 +93,7 @@ def get_toolchain(self, name, version=None): def test_toolchain(self): """Test whether toolchain is initialized correctly.""" - test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs') - ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb')) + ec_file = find_full_path(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb')) ec = EasyConfig(ec_file, validate=False) tc = ec.toolchain self.assertIn('debug', tc.options) @@ -860,8 +860,7 @@ def test_compiler_dependent_optarch(self): def test_easyconfig_optarch_flags(self): """Test whether specifying optarch flags in the easyconfigs works.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb') + eb_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') toy_txt = read_file(eb_file) @@ -2424,11 +2423,8 @@ def test_pgi_imkl(self): def test_compiler_cache(self): """Test ccache""" - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - args = [ - eb_file, + TOY_EC, "--use-ccache=%s" % os.path.join(self.test_prefix, 'ccache'), "--force", "--debug", @@ -3346,8 +3342,7 @@ def prep(): tmp_modules = os.path.join(self.test_prefix, 'modules') mkdir(tmp_modules) - test_dir = os.path.abspath(os.path.dirname(__file__)) - copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) + copy_dir(os.path.join(TEST_MODULES_DIR, 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '2.1.2-GCC-6.4.0-2.28') ompi_mod_txt = read_file(openmpi_module) @@ -3481,11 +3476,7 @@ def test_toolchain_external(self): write_file(tc_ec, tc_ec_txt) self.eb_main([tc_ec], raise_error=True, do_build=True) - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - - test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", toy_ec_txt) + test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", TOY_EC_TXT) test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e14ad542b5..18290137f8 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -42,10 +42,11 @@ import filecmp from easybuild.tools import LooseVersion from importlib import reload + from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup from test.framework.package import mock_fpm from unittest import TextTestRunner - +from test.framework import REPO_ROOT, TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT import easybuild.tools.hooks # so we can reset cached hooks import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -63,11 +64,6 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_ECS_DIR = os.path.join(TEST_DIR, 'easyconfigs', 'test_ecs') -TOY_EC = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0.eb') -TOY_EC_TXT: str = read_file(TOY_EC) - class ToyBuildTest(EnhancedTestCase): """Toy build unit test.""" @@ -176,7 +172,7 @@ def _test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=Tru args = [ ec_file, '--unittest-file=%s' % self.logfile, - '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, str(TEST_DIR)]), ] if debug: args.append('--debug') @@ -472,14 +468,14 @@ def test_toy_build_formatv2(self): """Perform a toy build (format v2).""" # set $MODULEPATH such that modules for specified dependencies are found modulepath = os.environ.get('MODULEPATH') - os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') + os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR) args = [ os.path.join(TEST_DIR, 'easyconfigs', 'v2.0', 'toy.eb'), '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, str(TEST_DIR)]), '--software-version=0.0', '--toolchain=system,system', '--experimental', @@ -544,7 +540,7 @@ def test_toy_build_formatv2_sections(self): '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, str(TEST_DIR)]), '--software-version=%s' % version, '--toolchain=system,system', '--experimental', @@ -1186,7 +1182,7 @@ def test_toy_hierarchical_subdir_user_modules(self): def test_toy_advanced(self): """Test toy build with extensions and non-system toolchain.""" - os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') + os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR) test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') with self.mocked_stdout_stderr(): self._test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=['--debug']) @@ -1221,7 +1217,7 @@ def test_toy_advanced_filter_deps(self): """Test toy build with extensions, and filtered build dependency.""" # test case for bug https://github.com/easybuilders/easybuild-framework/pull/2515 - os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') + os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR) toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_txt = read_file(toy_ec) @@ -1774,7 +1770,7 @@ def test_module_only(self): toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') # only consider provided test modules - self.reset_modulepath([os.path.join(TEST_DIR, 'modules')]) + self.reset_modulepath([os.path.join(TEST_MODULES_DIR)]) # sanity check fails without --force if software is not installed yet common_args = [ @@ -3016,8 +3012,7 @@ def test_toy_rpath(self): # find_eb_script function used to find rpath_args.py requires that location where easybuild/scripts # resides is listed in sys.path via absolute path; # this is only needed to make this test pass when it's being called from that same location... - top_path = os.path.dirname(os.path.dirname(TEST_DIR)) - sys.path.insert(0, top_path) + sys.path.insert(0, str(REPO_ROOT)) def grab_gcc_rpath_wrapper_args(): """Helper function to grab arguments from last RPATH wrapper for 'gcc'.""" @@ -4787,7 +4782,7 @@ def pre_configure_hook(self, *args, **kwargs): toy_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') args = [ - TOY_EC, + str(TOY_EC), f'--hooks={hooks_file}', '--force', f'--installpath={self.test_prefix}', diff --git a/test/framework/tweak.py b/test/framework/tweak.py index b59fbd4557..eb4dfe3f4d 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -29,6 +29,7 @@ """ import os import sys +from test.framework import TEST_ECS_DIR, TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -74,12 +75,11 @@ def test_pick_version(self): def test_find_matching_easyconfigs(self): """Test find_matching_easyconfigs function.""" - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') for (name, installver) in [('GCC', '4.8.2'), ('gzip', '1.5-foss-2018a')]: - ecs = find_matching_easyconfigs(name, installver, [test_easyconfigs_path]) + ecs = find_matching_easyconfigs(name, installver, [TEST_ECS_DIR]) self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver))) - ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path]) + ecs = find_matching_easyconfigs('GCC', '*', [TEST_ECS_DIR]) gccvers = ['10.2.0', '12.3.0', '4.6.3', '4.6.4', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25', '4.9.3-2.26', '6.4.0-2.28', '7.3.0-2.30'] self.assertEqual(len(ecs), len(gccvers)) @@ -92,14 +92,13 @@ def test_obtain_ec_for(self): """Test obtain_ec_for function.""" init_config(build_options={'silent': True}) - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # find existing easyconfigs specs = { 'name': 'GCC', 'version': '6.4.0', 'versionsuffix': '-2.28', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-6.4.0-2.28.eb') @@ -110,7 +109,7 @@ def test_obtain_ec_for(self): 'toolchain_version': '2018a', 'versionsuffix': '-OpenBLAS-0.2.20', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb') @@ -118,7 +117,7 @@ def test_obtain_ec_for(self): 'name': 'ifort', 'versionsuffix': '-GCC-4.9.3-2.25', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'ifort-2016.1.150-GCC-4.9.3-2.25.eb') @@ -126,7 +125,7 @@ def test_obtain_ec_for(self): specs = { 'name': 'GCC', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertFalse(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-12.3.0.eb') @@ -136,48 +135,45 @@ def test_obtain_ec_for(self): 'name': 'GCC', 'version': '4.9.0', } - (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + (generated, ec_file) = obtain_ec_for(specs, [TEST_ECS_DIR]) self.assertTrue(generated) self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.0.eb') def test_tweak_one_version(self): """Test tweak_one function""" - test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb') # test tweaking of software version (--try-software-version) - tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb') - tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + tweaked_TOY_EC = os.path.join(self.test_prefix, 'toy-tweaked.eb') + tweak_one(TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) - toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict() - tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() + TOY_EC_parsed = EasyConfigParser(TOY_EC).get_config_dict() + tweaked_TOY_EC_parsed = EasyConfigParser(tweaked_TOY_EC).get_config_dict() # checksums should be reset to empty list, only version should be changed, nothing else - self.assertEqual(tweaked_toy_ec_parsed['checksums'], []) - self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') - for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: - val = toy_ec_parsed[key] - self.assertIn(key, tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) - tweaked_val = tweaked_toy_ec_parsed.get(key) + self.assertEqual(tweaked_TOY_EC_parsed['checksums'], []) + self.assertEqual(tweaked_TOY_EC_parsed['version'], '1.2.3') + for key in [k for k in TOY_EC_parsed.keys() if k not in ['checksums', 'version']]: + val = TOY_EC_parsed[key] + self.assertIn(key, tweaked_TOY_EC_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) + tweaked_val = tweaked_TOY_EC_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) # check behaviour if target file already exists error_pattern = "File exists, not overwriting it without --force" - self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) # existing file does get overwritten when --force is used init_config(build_options={'force': True, 'silent': True}) - write_file(tweaked_toy_ec, '') - tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) - tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() - self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') + write_file(tweaked_TOY_EC, '') + tweak_one(TOY_EC, tweaked_TOY_EC, {'version': '1.2.3'}) + tweaked_TOY_EC_parsed = EasyConfigParser(tweaked_TOY_EC).get_config_dict() + self.assertEqual(tweaked_TOY_EC_parsed['version'], '1.2.3') def test_check_capability_mapping(self): """Test comparing the functionality of two toolchains""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), }) get_toolchain_hierarchy.clear() foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True) @@ -206,9 +202,8 @@ def test_check_capability_mapping(self): def test_match_minimum_tc_specs(self): """Test matching a toolchain to lowest possible in a hierarchy""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': module_classes(), }) @@ -244,10 +239,9 @@ def test_match_minimum_tc_specs(self): def test_dep_tree_of_toolchain(self): """Test getting list of dependencies of a toolchain (as EasyConfig objects)""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'check_osdeps': False, }) toolchain_spec = {'name': 'foss', 'version': '2018a'} @@ -267,9 +261,8 @@ def test_dep_tree_of_toolchain(self): def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': test_easyconfigs, + 'robot_path': str(TEST_ECS_DIR), 'silent': True, 'valid_module_classes': module_classes(), }) @@ -311,16 +304,15 @@ def test_map_toolchain_hierarchies(self): def test_get_matching_easyconfig_candidates(self): """Test searching for easyconfig candidates based on a stub and toolchain""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], }) toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} paths, toolchain_suff = get_matching_easyconfig_candidates('gzip-', toolchain) expected_toolchain_suff = '-GCC-4.9.3-2.26' self.assertEqual(toolchain_suff, expected_toolchain_suff) - expected_paths = [os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_suff + '.eb')] + expected_paths = [os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_suff + '.eb')] self.assertEqual(paths, expected_paths) paths, toolchain_stub = get_matching_easyconfig_candidates('nosuchmatch', toolchain) @@ -329,9 +321,8 @@ def test_get_matching_easyconfig_candidates(self): def test_map_common_versionsuffixes(self): """Test mapping between two toolchain hierarchies""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), }) @@ -355,9 +346,8 @@ def test_map_common_versionsuffixes(self): def test_find_potential_version_mappings(self): """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), }) @@ -367,7 +357,7 @@ def test_find_potential_version_mappings(self): iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) - ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') parsed_ec = process_easyconfig(ec_spec)[0] gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'][0] self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26') @@ -376,7 +366,7 @@ def test_find_potential_version_mappings(self): self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain expected = { - 'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), + 'path': os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), 'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'version': '1.6', 'versionsuffix': '', @@ -408,7 +398,7 @@ def test_find_potential_version_mappings(self): potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True) self.assertEqual(len(potential_versions), 1) expected = { - 'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'), + 'path': os.path.join(TEST_ECS_DIR, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'), 'toolchain': {'version': '6.4.0-2.28', 'name': 'GCC'}, 'version': '0.2.20', 'versionsuffix': '', @@ -417,9 +407,8 @@ def test_find_potential_version_mappings(self): def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } @@ -430,7 +419,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) - ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() @@ -497,7 +486,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): # Check that if we update a software version, it also updates the version if the software appears in an # extension list (like for a PythonBundle) - ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # Create the trivial toolchain mapping toolchain = {'name': 'gompi', 'version': '2018a'} tc_mapping = map_toolchain_hierarchies(toolchain, toolchain, self.modtool) @@ -523,20 +512,19 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): def test_list_deps_versionsuffixes(self): """Test listing of dependencies' version suffixes""" - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { - 'robot_path': [test_easyconfigs], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) get_toolchain_hierarchy.clear() - ec_spec = os.path.join(test_easyconfigs, 'g', 'golf', 'golf-2018a.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'g', 'golf', 'golf-2018a.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), ['-serial']) - ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), []) - ec_spec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + ec_spec = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') self.assertEqual(list_deps_versionsuffixes(ec_spec), ['-deps']) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index b698ea013a..07b9334cde 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,7 +38,9 @@ import unittest from contextlib import contextmanager from importlib import reload +from pathlib import Path +from test.framework import TEST_DIR, TEST_ECS_DIR, TEST_MODULES_DIR from easybuild.base import fancylogger from easybuild.base.testing import TestCase import easybuild.tools.build_log as eb_build_log @@ -121,10 +123,8 @@ def setUp(self): # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] - testdir = os.path.dirname(os.path.abspath(__file__)) - - self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') - self.test_sourcepath_data = os.path.join(testdir, 'sandbox', 'data_sources') + self.test_sourcepath = os.path.join(TEST_DIR, 'sandbox', 'sources') + self.test_sourcepath_data = os.path.join(TEST_DIR, 'sandbox', 'data_sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_SOURCEPATH_DATA'] = self.test_sourcepath_data os.environ['EASYBUILD_PREFIX'] = self.test_prefix @@ -134,7 +134,7 @@ def setUp(self): os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests - os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') + os.environ['EASYBUILD_ROBOT_PATHS'] = str(TEST_ECS_DIR) # make sure that the EasyBuild installation is still known even if we purge an EB module if os.getenv('EB_SCRIPT_PATH') is None: @@ -169,7 +169,7 @@ def setUp(self): pass # add sandbox to Python search path, update namespace packages - testdir_sandbox = os.path.join(testdir, 'sandbox') + testdir_sandbox = os.path.join(TEST_DIR, 'sandbox') sys.path.append(testdir_sandbox) # required to make sure the 'easybuild' dir in the sandbox is picked up; @@ -214,7 +214,7 @@ def setUp(self): self.env_pythonpath = os.environ.get('PYTHONPATH') self.modtool: ModulesTool = modules_tool() - self.reset_modulepath([os.path.join(testdir, 'modules')]) + self.reset_modulepath([os.path.join(TEST_MODULES_DIR)]) reset_module_caches() def disallow_deprecated_behaviour(self): @@ -316,7 +316,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal # always run main in unit testing mode (which for example allows for using deprecated toolchains); # note: don't change 'args' value, which is passed by reference! - main_args = args + ['--unit-testing-mode'] + main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode'] myerr = False if logfile is None: @@ -383,7 +383,7 @@ def setup_hierarchical_modules(self): # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler', 'MPI']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules', mod_subdir) + src_mod_path = os.path.join(TEST_MODULES_DIR, mod_subdir) copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir)) # make sure only modules in a hierarchical scheme are available, mixing modules installed with @@ -428,8 +428,7 @@ def setup_categorized_hmns_modules(self): # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler', 'MPI']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'modules', 'CategorizedHMNS', mod_subdir) + src_mod_path = os.path.join(TEST_MODULES_DIR, 'CategorizedHMNS', mod_subdir) copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir)) # create empty module file directory to make Environment Modules <5.0 happy mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2')