[2/6] Use constants for commonly used test easyconfig folders, files and content

easybuild/tools/filetools.py

@@ -61,6 +61,7 @@
 import zlib
 from functools import partial
 from html.parser import HTMLParser
+from pathlib import Path
 import urllib.request as std_urllib
 
 from easybuild.base import fancylogger
@@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs):
     :param force_in_dry_run: force running the command during dry run
     :param kwargs: additional named arguments to pass down to copy_dir
     """
-    if isinstance(paths, str):
+    if isinstance(paths, (str, Path)):
         paths = [paths]
 
     _log.info("Copying %d files & directories to %s", len(paths), target_path)

test/framework/__init__.py

@@ -27,3 +27,13 @@
 
 @author: Toon Willems (Ghent University)
 """
+
+from pathlib import Path
+from easybuild.tools.filetools import read_file
+
+TEST_DIR = Path(__file__).parent
+REPO_ROOT = TEST_DIR.parent.parent
+TEST_MODULES_DIR = TEST_DIR / 'modules'
+TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs'
+TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb'
+TOY_EC_TXT: str = read_file(TOY_EC)

test/framework/containers.py

@@ -31,6 +31,7 @@
 import re
 import stat
 import sys
+from test.framework import TEST_ECS_DIR, TOY_EC
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from unittest import TextTestRunner
 
@@ -89,9 +90,6 @@ def check_regexs(self, regexs, stdout):
 
     def test_end2end_singularity_recipe_config(self):
         """End-to-end test for --containerize (recipe only), using --container-config."""
-        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
-        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
-
         containerpath = os.path.join(self.test_prefix, 'containers')
         os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
         # --containerpath must be an existing directory (this is done to avoid misconfiguration)
@@ -100,7 +98,7 @@ def test_end2end_singularity_recipe_config(self):
         test_container_recipe = os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0')
 
         args = [
-            toy_ec,
+            TOY_EC,
             '--containerize',
             '--experimental',
         ]
@@ -257,9 +255,6 @@ def test_end2end_singularity_recipe_config(self):
 
     def test_end2end_singularity_image(self):
         """End-to-end test for --containerize (recipe + image)."""
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-
         containerpath = os.path.join(self.test_prefix, 'containers')
         os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
         # --containerpath must be an existing directory (this is done to avoid misconfiguration)
@@ -269,7 +264,7 @@ def test_end2end_singularity_image(self):
         write_file(test_img, '')
 
         args = [
-            toy_ec,
+            TOY_EC,
             '-C',  # equivalent with --containerize
             '--experimental',
             '--container-config=bootstrap=localimage,from=%s' % test_img,
@@ -360,16 +355,13 @@ def test_end2end_singularity_image(self):
         self.check_regexs(regexs, stdout)
 
     def test_end2end_dockerfile(self):
-        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
-        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
-
         containerpath = os.path.join(self.test_prefix, 'containers')
         os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
         # --containerpath must be an existing directory (this is done to avoid misconfiguration)
         mkdir(containerpath)
 
         base_args = [
-            toy_ec,
+            TOY_EC,
             '--containerize',
             '--container-type=docker',
             '--experimental',
@@ -402,7 +394,7 @@ def test_end2end_dockerfile(self):
 
         remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
 
-        base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb'))
+        base_args.insert(1, os.path.join(TEST_ECS_DIR, 'g', 'GCC', 'GCC-4.9.2.eb'))
         self.run_main(base_args + ['--container-config=ubuntu:20.04'])
         def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
         regexs = [
@@ -419,16 +411,13 @@ def test_end2end_dockerfile(self):
 
     def test_end2end_docker_image(self):
 
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-
         containerpath = os.path.join(self.test_prefix, 'containers')
         os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
         # --containerpath must be an existing directory (this is done to avoid misconfiguration)
         mkdir(containerpath)
 
         args = [
-            toy_ec,
+            TOY_EC,
             '-C',  # equivalent with --containerize
             '--experimental',
             '--container-type=docker',

test/framework/docs.py

@@ -39,6 +39,7 @@
 from easybuild.tools.options import EasyBuildOptions
 from easybuild.tools.utilities import mk_md_table, mk_rst_table
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
+from test.framework import TEST_DIR, TEST_ECS_DIR
 
 
 LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock
@@ -869,8 +870,7 @@ def test_list_easyblocks(self):
         """
         Tests for list_easyblocks function
         """
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        topdir_easyblocks = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks')
+        topdir_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks')
 
         txt = list_easyblocks()
         self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_TXT)
@@ -899,7 +899,7 @@ def test_list_easyblocks(self):
     def test_list_software(self):
         """Test list_software* functions."""
         build_options = {
-            'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')],
+            'robot_path': [os.path.join(TEST_DIR, 'easyconfigs', 'v1.0')],
             'silent': True,
             'valid_module_classes': module_classes(),
         }
@@ -933,7 +933,7 @@ def test_list_software(self):
 
         # check for specific patterns in output for larger set of test easyconfigs
         build_options = {
-            'robot_path': [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')],
+            'robot_path': [TEST_ECS_DIR],
             'silent': True,
             'valid_module_classes': module_classes(),
         }

test/framework/include.py

@@ -29,6 +29,7 @@
 """
 import os
 import sys
+from test.framework import TEST_DIR
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from unittest import TextTestRunner
 
@@ -53,7 +54,7 @@ class IncludeTest(EnhancedTestCase):
 
     def test_include_easyblocks(self):
         """Test include_easyblocks()."""
-        test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks')
+        test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks')
 
         # put a couple of custom easyblocks in place, to test
         myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks')
@@ -119,7 +120,7 @@ def test_include_easyblocks(self):
 
     def test_include_easyblocks_priority(self):
         """Test whether easyblocks included via include_easyblocks() get priority over others."""
-        test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks')
+        test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks')
 
         # make sure that test 'foo' easyblock is there
         import easybuild.easyblocks.foo
@@ -288,8 +289,7 @@ def test_is_software_specific_easyblock(self):
 
         self.assertErrorRegex(EasyBuildError, "No such file", is_software_specific_easyblock, '/no/such/easyblock.py')
 
-        testdir = os.path.dirname(os.path.abspath(__file__))
-        test_easyblocks = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks')
+        test_easyblocks = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks')
 
         self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 'g', 'gcc.py')))
         self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 't', 'toy.py')))

test/framework/package.py

@@ -33,6 +33,7 @@
 import sys
 
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
+from test.framework import TEST_ECS_DIR
 from unittest import TextTestRunner
 
 from easybuild.framework.easyconfig.easyconfig import EasyConfig
@@ -181,9 +182,7 @@ def test_active_pns(self):
             os.environ['EASYBUILD_PACKAGE_NAMING_SCHEME'] = pns_type
             init_config(build_options={'silent': True})
 
-            topdir = os.path.dirname(os.path.abspath(__file__))
-            test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
-            test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb')
+            test_ec = os.path.join(TEST_ECS_DIR, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb')
             ec = EasyConfig(test_ec, validate=False)
 
             pns = ActivePNS()
@@ -206,9 +205,7 @@ def test_package(self):
         }
         init_config(build_options=build_options)
 
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
-        ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False)
+        ec = EasyConfig(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False)
 
         mock_fpm(self.test_prefix)
 
@@ -246,7 +243,7 @@ def test_package(self):
         res = no_logfiles_regex.search(pkgtxt)
         self.assertFalse(res, "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt))
 
-        toy_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'))
+        toy_txt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'))
         replace_str = '''description = """Toy C program, 100% toy. Now with `backticks'\n'''
         replace_str += '''and newlines"""'''
         toy_txt = re.sub('description = .*', replace_str, toy_txt)

test/framework/repository.py

@@ -32,6 +32,7 @@
 import shutil
 import sys
 import tempfile
+from test.framework import TOY_EC
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from unittest import TextTestRunner, mock
 
@@ -100,8 +101,7 @@ def test_gitrepo(self):
         if res.exit_code == 0:
             repo = GitRepository(os.path.join(tmpdir, 'testrepository.git'))
             repo.init()
-            toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-            repo.add_easyconfig(toy_ec_file, 'test', '1.0', {}, None)
+            repo.add_easyconfig(TOY_EC, 'test', '1.0', {}, None)
             with mock.patch.dict(os.environ, {'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@test.org',
                                               'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@test.org'}):
                 repo.commit("toy/0.0")
@@ -146,7 +146,6 @@ def test_init_repository(self):
     def test_add_easyconfig(self):
         """Test use of add_easyconfig method"""
         repo = init_repository('FileRepository', self.path)
-        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
 
         def check_ec(path, expected_buildstats):
             """Check easyconfig at specified path"""
@@ -157,12 +156,10 @@ def check_ec(path, expected_buildstats):
             ecdict = EasyConfigParser(path).get_config_dict()
             self.assertEqual(ecdict['buildstats'], expected_buildstats)
 
-        toy_eb_file = os.path.join(test_easyconfigs, 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-
-        path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23}, None)
+        path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23}, None)
         check_ec(path, [{'time': 1.23}])
 
-        path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}])
+        path = repo.add_easyconfig(TOY_EC, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}])
         check_ec(path, [{'time': 0.9, 'size': 2}, {'time': 1.23, 'size': 123}])
 
     def tearDown(self):

test/framework/toolchain.py

@@ -37,6 +37,7 @@
 import textwrap
 from itertools import product
 from unittest import TextTestRunner
+from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC, TOY_EC_TXT
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config
 
 import easybuild.tools.modules as modules
@@ -92,8 +93,7 @@ def get_toolchain(self, name, version=None):
 
     def test_toolchain(self):
         """Test whether toolchain is initialized correctly."""
-        test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')
-        ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb'))
+        ec_file = find_full_path(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb'))
         ec = EasyConfig(ec_file, validate=False)
         tc = ec.toolchain
         self.assertIn('debug', tc.options)
@@ -860,8 +860,7 @@ def test_compiler_dependent_optarch(self):
 
     def test_easyconfig_optarch_flags(self):
         """Test whether specifying optarch flags in the easyconfigs works."""
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb')
+        eb_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb')
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         toy_txt = read_file(eb_file)
@@ -2424,11 +2423,8 @@ def test_pgi_imkl(self):
 
     def test_compiler_cache(self):
         """Test ccache"""
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-
         args = [
-            eb_file,
+            TOY_EC,
             "--use-ccache=%s" % os.path.join(self.test_prefix, 'ccache'),
             "--force",
             "--debug",
@@ -3346,8 +3342,7 @@ def prep():
         tmp_modules = os.path.join(self.test_prefix, 'modules')
         mkdir(tmp_modules)
 
-        test_dir = os.path.abspath(os.path.dirname(__file__))
-        copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI'))
+        copy_dir(os.path.join(TEST_MODULES_DIR, 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI'))
 
         openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '2.1.2-GCC-6.4.0-2.28')
         ompi_mod_txt = read_file(openmpi_module)
@@ -3481,11 +3476,7 @@ def test_toolchain_external(self):
         write_file(tc_ec, tc_ec_txt)
         self.eb_main([tc_ec], raise_error=True, do_build=True)
 
-        topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
-        toy_ec_txt = read_file(toy_ec)
-
-        test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", toy_ec_txt)
+        test_ec_txt = re.sub('toolchain.*', "toolchain = {'name': 'GCC', 'version': 'external'}", TOY_EC_TXT)
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         write_file(test_ec, test_ec_txt)

test/framework/utilities.py

@@ -38,6 +38,7 @@
 import unittest
 from contextlib import contextmanager
 from importlib import reload
+from pathlib import Path
 
 from easybuild.base import fancylogger
 from easybuild.base.testing import TestCase
@@ -316,7 +317,7 @@ def eb_main(self, args, do_build=False, return_error=False, return_exit_code=Fal
 
         # always run main in unit testing mode (which for example allows for using deprecated toolchains);
         # note: don't change 'args' value, which is passed by reference!
-        main_args = args + ['--unit-testing-mode']
+        main_args = [str(arg) if isinstance(arg, Path) else arg for arg in args] + ['--unit-testing-mode']
 
         myerr = False
         if logfile is None: