diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..120eb9640a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): :param force_in_dry_run: force running the command during dry run :param kwargs: additional named arguments to pass down to copy_dir """ - if isinstance(paths, str): + if isinstance(paths, (str, Path)): paths = [paths] _log.info("Copying %d files & directories to %s", len(paths), target_path) diff --git a/test/framework/__init__.py b/test/framework/__init__.py index 895b15e622..04daadf7fa 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -27,3 +27,13 @@ @author: Toon Willems (Ghent University) """ + +from pathlib import Path +from easybuild.tools.filetools import read_file + +TEST_DIR = Path(__file__).parent +REPO_ROOT = TEST_DIR.parent.parent +TEST_MODULES_DIR = TEST_DIR / 'modules' +TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs' +TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb' +TOY_EC_TXT: str = read_file(TOY_EC) diff --git a/test/framework/containers.py b/test/framework/containers.py index 93fc04548c..1c47ccfc45 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -31,6 +31,7 @@ import re import stat import sys +from test.framework import TEST_ECS_DIR, TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -89,9 +90,6 @@ def check_regexs(self, regexs, stdout): def test_end2end_singularity_recipe_config(self): """End-to-end test for --containerize (recipe only), using --container-config.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) @@ -100,7 +98,7 @@ def test_end2end_singularity_recipe_config(self): test_container_recipe = os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0') args = [ - toy_ec, + TOY_EC, '--containerize', '--experimental', ] @@ -257,9 +255,6 @@ def test_end2end_singularity_recipe_config(self): def test_end2end_singularity_image(self): """End-to-end test for --containerize (recipe + image).""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) @@ -269,7 +264,7 @@ def test_end2end_singularity_image(self): write_file(test_img, '') args = [ - toy_ec, + TOY_EC, '-C', # equivalent with --containerize '--experimental', '--container-config=bootstrap=localimage,from=%s' % test_img, @@ -360,16 +355,13 @@ def test_end2end_singularity_image(self): self.check_regexs(regexs, stdout) def test_end2end_dockerfile(self): - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) base_args = [ - toy_ec, + TOY_EC, '--containerize', '--container-type=docker', '--experimental', @@ -402,7 +394,7 @@ def test_end2end_dockerfile(self): remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) - base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb')) + base_args.insert(1, os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.2.eb')) self.run_main(base_args + ['--container-config=ubuntu:20.04']) def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) regexs = [ @@ -419,16 +411,13 @@ def test_end2end_dockerfile(self): def test_end2end_docker_image(self): - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) args = [ - toy_ec, + TOY_EC, '-C', # equivalent with --containerize '--experimental', '--container-type=docker', diff --git a/test/framework/docs.py b/test/framework/docs.py index b24138074e..72af821eda 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -39,6 +39,7 @@ from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_DIR, TEST_ECS_DIR LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock @@ -869,8 +870,7 @@ def test_list_easyblocks(self): """ Tests for list_easyblocks function """ - topdir = os.path.dirname(os.path.abspath(__file__)) - topdir_easyblocks = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + topdir_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') txt = list_easyblocks() self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_TXT) @@ -899,7 +899,7 @@ def test_list_easyblocks(self): def test_list_software(self): """Test list_software* functions.""" build_options = { - 'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')], + 'robot_path': [os.path.join(TEST_DIR, 'easyconfigs', 'v1.0')], 'silent': True, 'valid_module_classes': module_classes(), } @@ -933,7 +933,7 @@ def test_list_software(self): # check for specific patterns in output for larger set of test easyconfigs build_options = { - 'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')], + 'robot_path': [TEST_ECS_DIR], 'silent': True, 'valid_module_classes': module_classes(), } diff --git a/test/framework/include.py b/test/framework/include.py index 6be29381e4..fb77e9d21f 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -29,6 +29,7 @@ """ import os import sys +from test.framework import TEST_DIR from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner @@ -53,7 +54,7 @@ class IncludeTest(EnhancedTestCase): def test_include_easyblocks(self): """Test include_easyblocks().""" - test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # put a couple of custom easyblocks in place, to test myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks') @@ -119,7 +120,7 @@ def test_include_easyblocks(self): def test_include_easyblocks_priority(self): """Test whether easyblocks included via include_easyblocks() get priority over others.""" - test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') # make sure that test 'foo' easyblock is there import easybuild.easyblocks.foo @@ -288,8 +289,7 @@ def test_is_software_specific_easyblock(self): self.assertErrorRegex(EasyBuildError, "No such file", is_software_specific_easyblock, '/no/such/easyblock.py') - testdir = os.path.dirname(os.path.abspath(__file__)) - test_easyblocks = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks') self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 'g', 'gcc.py'))) self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 't', 'toy.py'))) diff --git a/test/framework/package.py b/test/framework/package.py index eb9b7e53c8..7a1326b12a 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -33,6 +33,7 @@ import sys from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework import TEST_ECS_DIR from unittest import TextTestRunner from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -181,9 +182,7 @@ def test_active_pns(self): os.environ['EASYBUILD_PACKAGE_NAMING_SCHEME'] = pns_type init_config(build_options={'silent': True}) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + test_ec = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') ec = EasyConfig(test_ec, validate=False) pns = ActivePNS() @@ -206,9 +205,7 @@ def test_package(self): } init_config(build_options=build_options) - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False) + ec = EasyConfig(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False) mock_fpm(self.test_prefix) @@ -246,7 +243,7 @@ def test_package(self): res = no_logfiles_regex.search(pkgtxt) self.assertFalse(res, "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt)) - toy_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')) + toy_txt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')) replace_str = '''description = """Toy C program, 100% toy. Now with `backticks'\n''' replace_str += '''and newlines"""''' toy_txt = re.sub('description = .*', replace_str, toy_txt) diff --git a/test/framework/repository.py b/test/framework/repository.py index a5e992235d..470ba3b787 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -32,6 +32,7 @@ import shutil import sys import tempfile +from test.framework import TOY_EC from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner, mock @@ -100,8 +101,7 @@ def test_gitrepo(self): if res.exit_code == 0: repo = GitRepository(os.path.join(tmpdir, 'testrepository.git')) repo.init() - toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - repo.add_easyconfig(toy_ec_file, 'test', '1.0', {}, None) + repo.add_easyconfig(TOY_EC, 'test', '1.0', {}, None) with mock.patch.dict(os.environ, {'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@test.org', 'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@test.org'}): repo.commit("toy/0.0") @@ -146,7 +146,6 @@ def test_init_repository(self): def test_add_easyconfig(self): """Test use of add_easyconfig method""" repo = init_repository('FileRepository', self.path) - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') def check_ec(path, expected_buildstats): """Check easyconfig at specified path""" @@ -157,12 +156,10 @@ def check_ec(path, expected_buildstats): ecdict = EasyConfigParser(path).get_config_dict() self.assertEqual(ecdict['buildstats'], expected_buildstats) - toy_eb_file = os.path.join(test_easyconfigs, 'test_ecs', 't', 'toy', 'toy-0.0.eb') - - path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23}, None) + path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23}, None) check_ec(path, [{'time': 1.23}]) - path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}]) + path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}]) check_ec(path, [{'time': 0.9, 'size': 2}, {'time': 1.23, 'size': 123}]) def tearDown(self): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index e1206e08a4..f87d7a0af7 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -37,6 +37,7 @@ import textwrap from itertools import product from unittest import TextTestRunner +from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config import easybuild.tools.modules as modules @@ -92,8 +93,7 @@ def get_toolchain(self, name, version=None): def test_toolchain(self): """Test whether toolchain is initialized correctly.""" - test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs') - ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb')) + ec_file = find_full_path(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb')) ec = EasyConfig(ec_file, validate=False) tc = ec.toolchain self.assertIn('debug', tc.options) @@ -860,8 +860,7 @@ def test_compiler_dependent_optarch(self): def test_easyconfig_optarch_flags(self): """Test whether specifying optarch flags in the easyconfigs works.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb') + eb_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') toy_txt = read_file(eb_file) @@ -2424,11 +2423,8 @@ def test_pgi_imkl(self): def test_compiler_cache(self): """Test ccache""" - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - args = [ - eb_file, + TOY_EC, "--use-ccache=%s" % os.path.join(self.test_prefix, 'ccache'), "--force", "--debug", @@ -3346,8 +3342,7 @@ def prep(): tmp_modules = os.path.join(self.test_prefix, 'modules') mkdir(tmp_modules) - test_dir = os.path.abspath(os.path.dirname(__file__)) - copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) + copy_dir(os.path.join(TEST_MODULES_DIR, 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '2.1.2-GCC-6.4.0-2.28') ompi_mod_txt = read_file(openmpi_module) @@ -3481,11 +3476,7 @@ def test_toolchain_external(self): write_file(tc_ec, tc_ec_txt) self.eb_main([tc_ec], raise_error=True, do_build=True) - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - - test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", toy_ec_txt) + test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", TOY_EC_TXT) test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index b698ea013a..dde290348e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,6 +38,7 @@ import unittest from contextlib import contextmanager from importlib import reload +from pathlib import Path from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -316,7 +317,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal # always run main in unit testing mode (which for example allows for using deprecated toolchains); # note: don't change 'args' value, which is passed by reference! - main_args = args + ['--unit-testing-mode'] + main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode'] myerr = False if logfile is None: