Skip to content

[4/6] Use constants for commonly used test easyconfig folders, files and content #5252

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Flamefire wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

[4/6] Use constants for commonly used test easyconfig folders, files and content #5252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3951dc
[4/6] Use constants for commonly used test easyconfig folders, files …
Flamefire Jun 18, 2026
10e3b84
Accept path args in test `eb_main`
Flamefire Aug 27, 2026
f21f192
Accept Path argument in `filetools.copy`
Flamefire Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion easybuild/tools/filetools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import zlib
from functools import partial
from html.parser import HTMLParser
from pathlib import Path
import urllib.request as std_urllib

from easybuild.base import fancylogger
Expand Down Expand Up @@ -2777,7 +2778,7 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs):
:param force_in_dry_run: force running the command during dry run
:param kwargs: additional named arguments to pass down to copy_dir
"""
if isinstance(paths, str):
if isinstance(paths, (str, Path)):
paths = [paths]

_log.info("Copying %d files & directories to %s", len(paths), target_path)
Expand Down
10 changes: 10 additions & 0 deletions test/framework/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,13 @@

@author: Toon Willems (Ghent University)
"""

from pathlib import Path
from easybuild.tools.filetools import read_file

TEST_DIR = Path(__file__).parent
REPO_ROOT = TEST_DIR.parent.parent
TEST_MODULES_DIR = TEST_DIR / 'modules'
TEST_ECS_DIR = TEST_DIR / 'easyconfigs' / 'test_ecs'
TOY_EC = TEST_ECS_DIR / 't' / 'toy' / 'toy-0.0.eb'
TOY_EC_TXT: str = read_file(TOY_EC)
33 changes: 15 additions & 18 deletions test/framework/module_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod
from easybuild.tools.utilities import quote_str
from test.framework import TEST_MODULES_DIR, TEST_ECS_DIR
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config


Expand All @@ -57,8 +58,7 @@ def setUp(self):
"""Test setup."""
super().setUp()
# find .eb file
topdir = os.path.dirname(os.path.abspath(__file__))
eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
eb_path = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)

Expand Down Expand Up @@ -801,9 +801,8 @@ def test_module_extensions(self):
# check if extensions option is enabled and some module extensions are defined
init_config(build_options={'module_extensions': True})

test_dir = os.path.abspath(os.path.dirname(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
os.environ['MODULEPATH'] = os.path.join(TEST_MODULES_DIR)
test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')

ec = EasyConfig(test_ec)
eb = EasyBlock(ec)
Expand Down Expand Up @@ -833,7 +832,7 @@ def test_module_extensions(self):
self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc))

# check if the extensions is missing if there are no extensions
test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb')
test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-test.eb')

ec = EasyConfig(test_ec)
eb = EasyBlock(ec)
Expand All @@ -849,7 +848,7 @@ def test_module_extensions(self):

# check if the extensions is missing if 'module_extensions' is disabled
init_config(build_options={'module_extensions': False})
test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')

ec = EasyConfig(test_ec)
eb = EasyBlock(ec)
Expand Down Expand Up @@ -1358,15 +1357,14 @@ def test_module_naming_scheme(self):
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})

ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(TEST_ECS_DIR) for fil in files]
# keep only easyconfig files (there may be additional files like patches, checksums.json, etc.)
ec_files = [x for x in ec_files if x.endswith('.eb')]

build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'robot_path': [ecs_dir],
'robot_path': [TEST_ECS_DIR],
'valid_stops': all_stops,
'validate': False,
}
Expand Down Expand Up @@ -1421,7 +1419,7 @@ def test_mns():
init_config(build_options=build_options)

err_pattern = 'nosucheasyconfigparameteravailable'
ec_file = os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
ec_file = os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
self.assertErrorRegex(EasyBuildError, err_pattern, EasyConfig, ec_file)

# test simple custom module naming scheme
Expand All @@ -1438,7 +1436,7 @@ def test_mns():
}
test_mns()

ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'))
ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/2018a')

# test module naming scheme using all available easyconfig parameters
Expand Down Expand Up @@ -1485,7 +1483,7 @@ def test_mns():
# determine full module name
self.assertEqual(ActiveMNS().det_full_module_name(dep_spec), ec2mod_map[dep_ec])

ec = EasyConfig(os.path.join(ecs_dir, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True)
ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'), hidden=True)
self.assertEqual(ec.full_mod_name, ec2mod_map['gzip-1.5-foss-2018a.eb'])
self.assertEqual(ec.toolchain.det_short_module_name(), 'foss/e69469ac250145c9e814e5dde93f5fde6d80375d')

Expand Down Expand Up @@ -1554,19 +1552,18 @@ def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""

moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain']
ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'robot_path': [TEST_ECS_DIR],
'valid_stops': all_stops,
'validate': False,
'valid_module_classes': moduleclasses,
}

def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0])
ec = EasyConfig(glob.glob(os.path.join(TEST_ECS_DIR, '*', '*', ecfile))[0])
self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
Expand Down Expand Up @@ -1652,7 +1649,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts,
test_ec(ecfile, *mns_vals)

# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb'))
ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec)

os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
Expand Down Expand Up @@ -1695,7 +1692,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts,
test_ec(ecfile, *mns_vals, init_modpaths=['Core/%s' % c for c in moduleclasses])

# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'i', 'impi', 'impi-5.1.2.150.eb'))
ec = EasyConfig(os.path.join(TEST_ECS_DIR, 'i', 'impi', 'impi-5.1.2.150.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec)

os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedModuleNamingScheme'
Expand Down
37 changes: 15 additions & 22 deletions test/framework/modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import shutil
import stat
import sys
from test.framework import TEST_ECS_DIR, TEST_MODULES_DIR, TOY_EC_TXT
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner

Expand Down Expand Up @@ -64,7 +65,7 @@ class ModulesTest(EnhancedTestCase):
def init_testmods(self, test_modules_paths=None):
"""Initialize set of test modules for test."""
if test_modules_paths is None:
test_modules_paths = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))]
test_modules_paths = [os.path.abspath(os.path.join(TEST_MODULES_DIR))]
self.reset_modulepath(test_modules_paths)

# for Lmod, this test has to run first, to avoid that it fails;
Expand All @@ -83,7 +84,7 @@ def test_long_module_path(self):
# copy one of the test modules there
gcc_mod_dir = os.path.join(long_mod_path, 'GCC')
os.makedirs(gcc_mod_dir)
gcc_mod_path = os.path.join(os.path.dirname(__file__), 'modules', 'GCC', '4.6.3')
gcc_mod_path = os.path.join(TEST_MODULES_DIR, 'GCC', '4.6.3')
copy_file(gcc_mod_path, gcc_mod_dir)

# try and use long modules path
Expand All @@ -97,8 +98,6 @@ def test_long_module_path(self):
def test_run_module(self):
"""Test for ModulesTool.run_module method."""

testdir = os.path.dirname(os.path.abspath(__file__))

for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']:
os.environ.pop(key, None)

Expand All @@ -107,7 +106,7 @@ def test_run_module(self):
self.modtool.run_module('load', 'GCC/6.4.0-2.28')
self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28')

self.reset_modulepath([os.path.join(testdir, 'modules')])
self.reset_modulepath([os.path.join(TEST_MODULES_DIR)])

self.assertNotIn('EBROOTGCC', os.environ)
self.modtool.run_module(['load', 'GCC/6.4.0-2.28'])
Expand Down Expand Up @@ -266,7 +265,7 @@ def test_exist(self):
self.assertEqual(self.modtool.exist(['OpenMPI'], maybe_partial=False, skip_avail=True), [False])

# exist works on hidden modules in Lua syntax (only with Lmod)
test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
test_modules_path = os.path.join(TEST_MODULES_DIR)
if isinstance(self.modtool, Lmod):
# make sure only the .lua module file is there, otherwise this test doesn't work as intended
self.assertExists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua'))
Expand Down Expand Up @@ -626,7 +625,7 @@ def test_prepend_module_path(self):
self.assertEqual(modulepath, curr_module_paths())

# prepending path that is 'deeper down' in $MODULEPATH works, brings it back to front
test_mods_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
test_mods_dir = os.path.join(TEST_MODULES_DIR)
self.assertTrue(any(os.path.samefile(test_mods_dir, p) for p in modulepath))
self.modtool.prepend_module_path(test_mods_dir)
self.assertTrue(os.path.samefile(curr_module_paths()[0], test_mods_dir))
Expand Down Expand Up @@ -771,7 +770,7 @@ def check_get_software_libdir(expected, **additional_args):

def test_wrong_modulepath(self):
"""Test whether modules tool can deal with a broken $MODULEPATH."""
test_modules_path = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules'))
test_modules_path = os.path.realpath(os.path.join(TEST_MODULES_DIR))
modules_test_installpath = os.path.join(self.test_installpath, 'modules', 'all')
os.environ['MODULEPATH'] = '/some/non-existing/path:/this/doesnt/exists/anywhere:%s' % test_modules_path
init_config()
Expand All @@ -785,16 +784,15 @@ def test_wrong_modulepath(self):

def test_modulefile_path(self):
"""Test modulefile_path method"""
test_dir = os.path.abspath(os.path.dirname(__file__))
gcc_mod_file = os.path.join(test_dir, 'modules', 'GCC', '6.4.0-2.28')
gcc_mod_file = os.path.join(TEST_MODULES_DIR, 'GCC', '6.4.0-2.28')

modtool = modules_tool()
res = modtool.modulefile_path('GCC/6.4.0-2.28')
self.assertTrue(os.path.samefile(res, gcc_mod_file))

if isinstance(self.modtool, Lmod):
res = modtool.modulefile_path('bzip2/.1.0.6')
self.assertTrue(os.path.samefile(res, os.path.join(test_dir, 'modules', 'bzip2', '.1.0.6.lua')))
self.assertTrue(os.path.samefile(res, os.path.join(TEST_MODULES_DIR, 'bzip2', '.1.0.6.lua')))
res = modtool.modulefile_path('bzip2/.1.0.6', strip_ext=True)
self.assertTrue(res.endswith('test/framework/modules/bzip2/.1.0.6'))

Expand All @@ -814,11 +812,10 @@ def test_path_to_top_of_module_tree(self):
def test_path_to_top_of_module_tree_hierarchical_mns(self):
"""Test function to determine path to top of the module tree for a hierarchical module naming scheme."""

ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'robot_path': [TEST_ECS_DIR],
'valid_stops': all_stops,
'validate': False,
}
Expand Down Expand Up @@ -853,7 +850,7 @@ def test_path_to_top_of_module_tree_lua(self):
"""Test path_to_top_of_module_tree function on modules in Lua syntax."""
if isinstance(self.modtool, Lmod):
orig_modulepath = os.environ.get('MODULEPATH')
self.modtool.unuse(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules'))
self.modtool.unuse(os.path.join(TEST_MODULES_DIR))
curr_modulepath = os.environ.get('MODULEPATH')
error_msg = "Incorrect $MODULEPATH value after unuse: %s (orig: %s)" % (curr_modulepath, orig_modulepath)
self.assertEqual(curr_modulepath, None, error_msg)
Expand Down Expand Up @@ -1024,11 +1021,10 @@ def test_path_to_top_of_module_tree_categorized_hmns(self):
scheme.
"""

ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'robot_path': [TEST_ECS_DIR],
'valid_stops': all_stops,
'validate': False,
}
Expand Down Expand Up @@ -1063,7 +1059,7 @@ def test_path_to_top_of_module_tree_categorized_hmns(self):

def test_modules_tool_stateless(self):
"""Check whether ModulesTool instance is stateless between runs."""
test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
test_modules_path = os.path.join(TEST_MODULES_DIR)

# copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load
copy_dir(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core'))
Expand Down Expand Up @@ -1159,7 +1155,7 @@ def test_module_caches(self):

# create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH
# invalidate_module_caches_for should be able to deal with this
test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
test_mods_path = os.path.join(TEST_MODULES_DIR)
mods_symlink = os.path.join(self.test_prefix, 'modules_symlink')
os.symlink(test_mods_path, mods_symlink)
self.modtool.use(mods_symlink)
Expand Down Expand Up @@ -1588,11 +1584,8 @@ def test_get_setenv_value_from_modulefile(self):

init_config(build_options={'generate_devel_module': True})

topdir = os.path.dirname(os.path.abspath(__file__))
eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')

test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, read_file(eb_path))
write_file(test_ec, TOY_EC_TXT)
write_file(test_ec, "\nmodextravars = {'FOO': 'value with spaces'}", append=True)

toy_eb = EasyBlock(EasyConfig(test_ec))
Expand Down
Loading
Loading