add BLAS test

eessi/testsuite/eessi_mixin.py

@@ -120,10 +120,10 @@ def EESSI_mixin_run_after_init(self):
         # Filter on which scales are supported by the partitions defined in the ReFrame configuration
         hooks.filter_supported_scales(self)
 
-        hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type)
-
         hooks.set_modules(self)
 
+        hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type)
+
         # Set scales as tags
         hooks.set_tag_scale(self)
 
@@ -201,10 +201,11 @@ def EESSI_mixin_log_runtime_info(self):
         path to the modulefile, EESSI software subdir, EESSI testsuite version"""
         self.postrun_cmds.append('echo "EESSI_CVMFS_REPO: $EESSI_CVMFS_REPO"')
         self.postrun_cmds.append('echo "EESSI_SOFTWARE_SUBDIR: $EESSI_SOFTWARE_SUBDIR"')
-        if self.module_name:
-            # Get full modulepath
-            get_full_modpath = f'echo "FULL_MODULEPATH: $(module --location show {self.module_name})"'
-            self.postrun_cmds.append(get_full_modpath)
+        if self.module_names:
+            for module in self.module_names:
+                # Get full modulepath
+                get_full_modpath = f'echo "FULL_MODULEPATH: $(module --location show {module})"'
+                self.postrun_cmds.append(get_full_modpath)
 
     @run_before('run', always_last=True)
     def EESSI_mixin_set_user_executable_opts(self):

eessi/testsuite/hooks.py

@@ -463,7 +463,7 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic
     cause the valid_systems to be set to an empty string, and consequently the
     test.valid_systems to an invalid system name (eessi.testsuite.constants.INVALID_SYSTEM).
     """
-    is_cuda_module = is_cuda_required_module(test.module_name)
+    is_cuda_module = is_cuda_required_module(test.module_names)
 
     if is_cuda_module and required_device_type == DEVICE_TYPES.GPU:
         # CUDA modules and when using a GPU require partitions with FEATURES.GPU feature and
@@ -599,14 +599,23 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: float):
 
 def set_modules(test: rfm.RegressionTest):
     """
-    Skip current test if module_name is not among a list of modules,
+    Set modules test parameter via module_name, which can be a string or a list of strings
+    Skip current test if any of the module names is not present in the list of modules,
     specified with --setvar modules=<comma-separated-list>.
     """
-    if test.modules and test.module_name not in test.modules:
-        test.valid_systems = []
-        log(f'valid_systems set to {test.valid_systems}')
-
-    test.modules = [test.module_name]
+    if not test.module_name:
+        return
+    if isinstance(test.module_name, str):
+        test.module_names = [test.module_name]
+    else:
+        test.module_names = test.module_name
+    if test.modules:
+        for name in test.module_names:
+            if name not in test.modules:
+                test.valid_systems = []
+                log(f'module {name} not in {test.modules}, valid_systems set to {test.valid_systems}')
+
+    test.modules = test.module_names
     log(f'modules set to {test.modules}')
 
 

eessi/testsuite/tests/libs/blas/__init__.py

@@ -0,0 +1,2 @@
+# WARNING: do not remove this file.
+# It is needed to autogenerate documentation from this repo.

eessi/testsuite/tests/libs/blas/blas.py

@@ -0,0 +1,272 @@
+"""
+This test is adapted from the BLAS test included in the BLIS v1.1 sources at
+https://github.com/flame/blis/tree/1.1/test/3
+
+Customizations to the original BLAS test:
+- adapted and simplified Makefile for FlexiBLAS support
+- custom simplified run.sh script
+
+Supported tags in this ReFrame test (in addition to the common tags):
+- threading: `st`, `mt`
+- BLAS implementation: `openblas`, `blis`, `aocl-blas`, `mkl`
+- toolchain: `2023a`, `2023b`, `2024a`, ...
+- `CI` tag: runs only openblas st + mt
+"""
+
+
+import reframe as rfm
+from reframe.core.backends import getlauncher
+from reframe.core.builtins import parameter, run_after, run_before, sanity_function
+import reframe.utility.sanity as sn
+
+from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES
+from eessi.testsuite.eessi_mixin import EESSI_Mixin
+from eessi.testsuite.utils import check_modules_avail, log
+
+BLAS_MODULES = {
+    '2023a': {
+        'FlexiBLAS': 'FlexiBLAS/3.3.1-GCC-12.3.0',
+        'BLIS': 'BLIS/0.9.0-GCC-12.3.0',
+        'imkl': 'imkl/2023.1.0',
+    },
+    '2023b': {
+        'FlexiBLAS': 'FlexiBLAS/3.3.1-GCC-13.2.0',
+        'BLIS': 'BLIS/1.0-GCC-13.2.0',
+        'imkl': 'imkl/2023.2.0',
+    },
+    '2024a': {
+        'FlexiBLAS': 'FlexiBLAS/3.4.4-GCC-13.3.0',
+        'BLIS': 'BLIS/1.0-GCC-13.3.0',
+        'AOCL-BLAS': 'AOCL-BLAS/5.0-GCC-13.3.0',
+        'imkl': 'imkl/2024.2.0',
+    },
+    '2025a': {
+        'FlexiBLAS': 'FlexiBLAS/3.4.5-GCC-14.2.0',
+        'BLIS': 'BLIS/1.1-GCC-14.2.0',
+        'AOCL-BLAS': 'AOCL-BLAS/5.0-GCC-14.2.0',
+        'imkl': 'imkl/2025.1.0',
+    },
+}
+
+# FlexiBLAS and BLIS must always be loaded, even if BLIS is not used
+BASE_BLAS_MODULES = ['FlexiBLAS', 'BLIS']
+
+
+def single_thread_scales():
+    """Scales for the single-threaded tests"""
+    return parameter(['1_core', '1_node'])
+
+
+def multi_thread_scales():
+    """Scales for the multi-threaded tests"""
+    return parameter([
+        k for (k, v) in SCALES.items()
+        if v['num_nodes'] == 1
+    ])
+
+
+def get_module_lists(req_modules):
+    """Get a list of lists of full module names that are required for the test and available on the system"""
+    req_modules = set(req_modules)
+    ml_lists = [
+        [x[y] for y in req_modules]
+        for x in BLAS_MODULES.values()
+        if req_modules.issubset(x.keys())
+    ]
+    return [x for x in ml_lists if check_modules_avail(x)]
+
+
+class EESSI_BLAS_base(rfm.RunOnlyRegressionTest):
+    "base BLAS test"
+    device_type = DEVICE_TYPES.CPU
+    compute_unit = COMPUTE_UNITS.NODE
+    time_limit = '10m'
+    readonly_files = ['Makefile', 'run.sh', 'test_gemm.c', 'test_hemm.c', 'test_herk.c', 'test_trmm.c', 'test_trsm.c',
+                      'test_utils.c', 'test_utils.h']
+    env_vars = {
+        'CFLAGS': '"-O2 -ftree-vectorize -march=native -fno-math-errno -g"',  # default EB CFLAGS
+    }
+    executable = './run.sh'
+    nrepeats = '3'
+    dts = ['s', 'd', 'c', 'z']
+    ops = ['gemm_nn', 'hemm_ll', 'herk_ln', 'trmm_llnn', 'trsm_runn']
+    sizes = {
+        'st': ['100', '1000', '100'],
+        'mt': ['200', '2000', '200'],
+    }
+
+    def required_mem_per_node(self):
+        return self.num_cpus_per_task * 100 + 250
+
+    @run_after('init')
+    def set_prerun_cmds(self):
+        """Set prerun_cmds"""
+        self.prerun_cmds = [f'make flexiblas-{self.threading}']
+
+    @run_after('init')
+    def _tags(self):
+        """Add tags for toolchain (e.g. 2024a) and threading (st or mt)"""
+        blas_modules = {x: set(y.values()) for x, y in BLAS_MODULES.items()}
+        module_names = set(self.module_name)
+        toolchain = [x for x, y in blas_modules.items() if (module_names & y == module_names)][0]
+        self.tags.add(toolchain)
+
+        self.tags.add(self.threading)
+        log(f'tags set to {self.tags}')
+
+    @run_after('init')
+    def set_executable_opts(self):
+        """Set executable_opts"""
+        self.size = self.sizes[self.threading]
+        self.executable_opts = [
+            self.threading,
+            self.nrepeats,
+            '"{}"'.format(' '.join(self.size)),
+            '"{}"'.format(' '.join(self.dts)),
+            '"{}"'.format(' '.join(self.ops)),
+        ]
+
+    @run_after('init')
+    def set_flexiblas_blas_lib(self):
+        """Set FLEXIBLAS environment variable to selected BLAS lib"""
+        self.env_vars.update({
+            'FLEXIBLAS': self.flexiblas_blas_lib,
+        })
+
+    @run_after('setup')
+    def set_launcher(self):
+        """Select local launcher"""
+        self.job.launcher = getlauncher('local')()
+
+    @sanity_function
+    def assert_sanity(self):
+        assert_backend = sn.assert_not_found(
+            r'BLAS backend\s+\S+\s+not found',
+            self.stderr,
+            f'FlexiBLAS BLAS backend not found ({self.flexiblas_blas_lib})'
+        )
+        asserts_result = [
+            sn.assert_found(r"data\S+_flexiblas", f'output/{x}{y}_flexiblas.m', f'output/{x}{y}_flexiblas.m')
+            for x in self.dts for y in self.ops
+        ]
+        return sn.all([assert_backend] + asserts_result)
+
+    def _extract_perf(self, dt, op):
+        return sn.extractsingle(
+            rf'^data\S+_flexiblas.*{self.size[1]}\s+(?P<perf>\S+)\s+\];',
+            f'output/{dt}{op}_flexiblas.m',
+            1, float
+        )
+
+    @run_before('performance')
+    def set_perf_vars(self):
+        """Set performance variables"""
+        self.perf_variables.update({
+            f'{x}{y.split("_")[0]}': sn.make_performance_function(self._extract_perf(x, y), 'GFLOPS')
+            for x in self.dts for y in self.ops
+        })
+
+
+class EESSI_BLAS_OpenBLAS_base(EESSI_BLAS_base):
+    "base OpenBLAS test"
+
+    module_lists = get_module_lists(BASE_BLAS_MODULES)
+    module_name = parameter(module_lists)
+
+    flexiblas_blas_lib = 'openblas'
+    tags = {'openblas'}
+
+
+@rfm.simple_test
+class EESSI_BLAS_OpenBLAS_st(EESSI_BLAS_OpenBLAS_base, EESSI_Mixin):
+    "single-threaded OpenBLAS test"
+
+    scale = single_thread_scales()
+
+    bench_name = bench_name_ci = 'OpenBLAS_st'
+    threading = 'st'
+
+
+@rfm.simple_test
+class EESSI_BLAS_OpenBLAS_mt(EESSI_BLAS_OpenBLAS_base, EESSI_Mixin):
+    "multi-threaded OpenBLAS test"
+
+    scale = multi_thread_scales()
+
+    bench_name = bench_name_ci = 'OpenBLAS_mt'
+    threading = 'mt'
+
+
+class EESSI_BLAS_AOCLBLAS_base(EESSI_BLAS_base):
+    "base AOCL-BLAS test"
+
+    module_lists = get_module_lists(BASE_BLAS_MODULES + ['AOCL-BLAS'])
+    module_name = parameter(module_lists)
+
+    flexiblas_blas_lib = 'aocl_mt'
+    tags = {'aocl-blas'}
+
+
+@rfm.simple_test
+class EESSI_BLAS_AOCLBLAS_st(EESSI_BLAS_AOCLBLAS_base, EESSI_Mixin):
+    "single-threaded AOCL-BLAS test"
+
+    scale = single_thread_scales()
+    threading = 'st'
+
+
+@rfm.simple_test
+class EESSI_BLAS_AOCLBLAS_mt(EESSI_BLAS_AOCLBLAS_base, EESSI_Mixin):
+    "multi-threaded AOCL-BLAS test"
+
+    scale = multi_thread_scales()
+    threading = 'mt'
+
+
+class EESSI_BLAS_MKL_base(EESSI_BLAS_base):
+    "base MKL test"
+
+    module_lists = get_module_lists(BASE_BLAS_MODULES + ['imkl'])
+    module_name = parameter(module_lists)
+
+    flexiblas_blas_lib = 'imkl'
+    tags = {'mkl'}
+
+
+@rfm.simple_test
+class EESSI_BLAS_MKL_st(EESSI_BLAS_MKL_base, EESSI_Mixin):
+    "single-threaded MKL test"
+
+    scale = single_thread_scales()
+    threading = 'st'
+
+
+@rfm.simple_test
+class EESSI_BLAS_MKL_mt(EESSI_BLAS_MKL_base, EESSI_Mixin):
+    "multi-threaded MKL test"
+
+    scale = multi_thread_scales()
+    threading = 'mt'
+
+
+class EESSI_BLAS_BLIS_base(EESSI_BLAS_OpenBLAS_base):
+    "base BLIS test"
+
+    flexiblas_blas_lib = 'blis'
+    tags = {'blis'}
+
+
+@rfm.simple_test
+class EESSI_BLAS_BLIS_st(EESSI_BLAS_BLIS_base, EESSI_Mixin):
+    "single-threaded BLIS test"
+
+    scale = single_thread_scales()
+    threading = 'st'
+
+
+@rfm.simple_test
+class EESSI_BLAS_BLIS_mt(EESSI_BLAS_BLIS_base, EESSI_Mixin):
+    "multi-threaded BLIS test"
+
+    scale = multi_thread_scales()
+    threading = 'mt'