diff --git a/test/easyblocks/general.py b/test/easyblocks/general.py index 06c81651763..f3d0f04e16b 100644 --- a/test/easyblocks/general.py +++ b/test/easyblocks/general.py @@ -27,15 +27,21 @@ @author: Kenneth Hoste (Ghent University) """ +import itertools import os +import re import shutil import sys import tempfile +from pathlib import Path from unittest import TestLoader, TextTestRunner +from typing import List, Set from easybuild.base.testing import TestCase from easybuild.easyblocks import VERSION +from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.run import run_shell_cmd @@ -103,7 +109,7 @@ def test_custom_easyblocks_repo(self): # this is required to dance around issues with easy-install.pth files determining the actual Python search path # see also # http://blog.olgabotvinnik.com/blog/2014/03/03/2014-03-03-pythonpath-is-a-liar-site-py-and-easy-install-pth-tell/ - import easybuild.framework + import easybuild.framework # pylint: disable=import-outside-toplevel framework_path = up(easybuild.framework.__file__, 3) # prepend path to easybuild-easyblocks repo to $PYTHONPATH, so we're in full(?) control @@ -116,9 +122,7 @@ def test_custom_easyblocks_repo(self): def write_module(path, txt): """Write provided contents to module at given path in custom easyblocks repo.""" - handle = open(os.path.join(custom_easyblocks_repo_path, 'easybuild', path), 'w') - handle.write(txt) - handle.close() + write_file(os.path.join(custom_easyblocks_repo_path, 'easybuild', path), txt) # this test should be run out of the easyblocks repository, # to avoid that the working directory that is prepended to the Python search path affects the test results @@ -180,6 +184,57 @@ def write_module(path, txt): # importing EB_R class from easybuild.easyblocks.r still works fine run_shell_cmd("python -c 'from easybuild.easyblocks.r import EB_R'", hidden=True) + def test_imports(self): + """Check that for correct imports of other easyblocks + + All imports should be like: + from easybuild.easyblocks import cmake + from easybuild.easyblocks.generic import pythonpackage + import easybuild.easyblocks.cmake + import easybuild.easyblocks.generic.pythonpackage + + NOT using the lettered imports (breaks --include-easyblocks*): import easybuild.easyblocks.c.cmake + """ + easyblocks_path = Path(get_paths_for("easyblocks")[0]) + easyblocks: List[Path] = [eb for eb in easyblocks_path.rglob('*.py') + if eb.name != '__init__.py' and eb.parent.name != 'test'] + easyblock_names: Set[str] = {eb.stem for eb in easyblocks} + generic_easyblocks: Set[str] = {eb.stem for eb in easyblocks if eb.parent.name == 'generic'} + failures: List[str] = [] + for eb in easyblocks: + # Search for all imports of easyblocks and extract the name + contents: str = read_file(eb) + imports = itertools.chain( + re.finditer(r'(from easybuild\.easyblocks(\.(?P[^ ]+))?) import (?P.+)', contents), + re.finditer(r'import (easybuild\.easyblocks\.(?P[^ ]+))', contents), + ) + for imp in imports: + module = (imp['module'] or '').split('.') + if module and module[-1] in easyblock_names: + imported_eb = module[-1] + if imported_eb in generic_easyblocks: + if module[:-1] != ['generic']: + failures.append(f"Wrong import of generic easyblock '{imported_eb}' in {eb.name}: {imp[1]}") + elif len(module) != 1: # Should import directly, i.e. not 'easybuild.easyblocks.p.python' + failures.append(f"Wrong import of custom easyblock '{imported_eb}' in {eb.name}: {imp[1]}") + else: + try: + names = imp['names'] + except IndexError: + continue + for imported_eb in re.split(', *', names or ''): + imported_eb = re.sub(r' as .+', '', imported_eb) + if imported_eb not in easyblock_names: + continue + if imported_eb in generic_easyblocks: + if module != ['generic']: + failures.append(f"Wrong import of generic easyblock '{imported_eb}' in {eb.name}: " + f"{imp[1]}") + elif module: # Should import directly, i.e. not 'from easybuild.easyblocks.p import python' + failures.append(f"Wrong import of custom easyblock '{imported_eb}' in {eb.name}: {imp[1]}") + if failures: + self.fail('\n'.join(failures)) + def suite(loader): """Return all general easybuild-easyblocks tests.""" diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py index fa71d09d31c..9f188a0f635 100644 --- a/test/easyblocks/init_easyblocks.py +++ b/test/easyblocks/init_easyblocks.py @@ -43,11 +43,11 @@ from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.tools import config from easybuild.tools.config import GENERAL_CLASS -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.options import set_tmpdir # these imports are required because of checks done in template_init_test -from easybuild.tools.environment import modify_env, read_environment # noqa -from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa # noqa +from easybuild.tools.environment import modify_env, read_environment # noqa, pylint: disable=unused-import +from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa # noqa, pylint: disable=unused-import class InitTest(TestCase): @@ -124,9 +124,7 @@ def check_extra_options_format(extra_options): self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module - f = open(easyblock, "r") - txt = f.read() - f.close() + txt = read_file(easyblock) regexps = [ # make sure error reporting is done correctly (no more log.error, log.exception) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 846c2c3ef33..50b645f73ab 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -292,7 +292,7 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt= os.environ['INTEL_LICENSE_FILE'] = os.path.join(tmpdir, 'intel.lic') write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license') - elif app_class == PythonBundle or app_class == CargoPythonBundle: + elif app_class in (PythonBundle, CargoPythonBundle): # $EBROOTPYTHON must be set for PythonBundle easyblock os.environ['EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a' diff --git a/test/easyblocks/suite.py b/test/easyblocks/suite.py index 44429fa9912..4dd6128c1e8 100755 --- a/test/easyblocks/suite.py +++ b/test/easyblocks/suite.py @@ -48,6 +48,7 @@ class EasyBuildEasyBlocksTestSuite(unittest.TestSuite): + """Combine sub-test suites into one and setup environment before execution""" def __init__(self, loader): super().__init__([x.suite(loader) for x in [g, i, m, e]]) @@ -85,7 +86,7 @@ def run(self, *args, **kwargs): return res -def load_tests(loader, tests, pattern): +def load_tests(loader, _tests, _pattern): return EasyBuildEasyBlocksTestSuite(loader)