Automated cmake test discovery

CMakeLists.txt

@@ -118,8 +118,14 @@ if (FORTUNO_INSTALL)
       ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfig.cmake
       INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Fortuno
   )
+  configure_file(
+    cmake/Fortuno.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/Fortuno.cmake
+    COPYONLY
+  )
   install(
     FILES
+      ${CMAKE_CURRENT_BINARY_DIR}/Fortuno.cmake
       ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfigVersion.cmake
       ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfig.cmake
     DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Fortuno
@@ -143,6 +149,9 @@ endif ()
 
 # Make project available for FetchContent
 if (NOT PROJECT_IS_TOP_LEVEL)
+  list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+  set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
+
   # Propagate variables
   if (CMAKE_VERSION VERSION_LESS 3.25)
     # TODO: Remove when required minimum cmake version is >= 3.25

README.rst

@@ -4,8 +4,8 @@ Fortuno – flextensible unit testing framework for Fortran
 
 **Fortuno** (Fortran Unit Testing Objects) is a flexible & extensible,
 object-oriented unit testing framework designed for the Fortran programming
-language. It emphasizes ease of use by minimizing boiler plate code when writing
-tests while prioratizing modularity and extensibility. Fortuno provides the
+language. It emphasizes ease of use by minimizing boilerplate code when writing
+tests while prioritizing modularity and extensibility. Fortuno provides the
 essential building blocks to help developers create customized unit testing
 solutions.
 
@@ -336,12 +336,17 @@ your chosen build system:
 * **CMake**:  In your ``CMakeLists.txt`` file, declare an executable ``testapp``
   using ``testapp.f90`` as the source file and add ``Fortuno::fortuno_serial``
   as a dependency. Be sure to also link your library (e.g. ``mylib``).
-  Additionally, register the executable as a test, so that it can be executed
-  with ``ctest``::
+  Finally, use ``fortuno_discover_tests()`` to register the tests, so that they
+  can be executed with ``ctest``::
 
     add_executable(testapp testapp.f90)
     target_link_libraries(testapp PRIVATE mylib Fortuno::fortuno_serial)
-    add_test(NAME factorial COMMAND testapp)
+    fortuno_discover_tests(testapp)
+
+  *Note*: Alternatively, you can register the test executable manually with
+  ``add_test(NAME factorial COMMAND testapp)``. However,
+  ``fortuno_discover_tests()`` is recommended as it automatically discovers and
+  registers all tests in the executable.
 
   *Note*:  If you are using the MPI or coarray interface, replace
   ``Fortuno::fortuno_serial`` with ``Fortuno::fortuno_mpi`` or

cmake/Fortuno.cmake

@@ -0,0 +1,146 @@
+include_guard(GLOBAL)
+# Usage:
+# fortuno_discover_tests(target,
+#     [USE_PRE_TEST_DISCOVERY] # (Optional) flag to use pre-test discovery mode (default is post-build)
+#     [TEST_PREFIX prefix]     # (Optional) prefix for test names
+#     [TEST_SUFFIX suffix]     # (Optional) suffix for test names
+#     [TEST_PATTERN pattern]   # (Optional) pattern to filter tests (default is none, i.e. all tests)
+#     [WORKING_DIRECTORY dir]  # (Optional) working directory for tests (default is CMAKE_CURRENT_BINARY_DIR)
+# Description:
+# Query the Fortuno test executable for its list of tests and individually register them with CTest.
+# Querying is performed either at build time (default POST_BUILD mode) or at test time (PRE_TEST_DISCOVERY mode).
+# In the former case, an additional testlist.cmake file is generated after building the target to be included by CTest.
+# In the latter case, the test discovery is performed anew each time tests are run.
+function(fortuno_discover_tests target)
+    if (NOT TARGET ${target})
+        message(FATAL_ERROR "fortuno_discover_tests: target '${target}' does not exist")
+    endif ()
+
+    # Parse arguments and apply defaults
+    cmake_parse_arguments(ARG "USE_PRE_TEST_DISCOVERY" "TEST_PREFIX;TEST_SUFFIX;TEST_PATTERN;WORKING_DIRECTORY" "" ${ARGN})
+
+    if (ARG_UNPARSED_ARGUMENTS)
+        message(FATAL_ERROR "fortuno_discover_tests: Unknown arguments: ${ARG_UNPARSED_ARGUMENTS}")
+    endif ()
+
+    if (NOT DEFINED ARG_WORKING_DIRECTORY)
+        set(ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+    endif ()
+
+
+    set(discovery_script "${CMAKE_CURRENT_BINARY_DIR}/discover_tests_${target}_$<CONFIG>.cmake")
+    set(dispatch_script "${CMAKE_CURRENT_BINARY_DIR}/dispatch_discover_${target}.cmake")
+    set(testlist_file "${CMAKE_CURRENT_BINARY_DIR}/testlist_${target}_$<CONFIG>.cmake")
+
+    # Point ctest to the dispatch script which either loads the testlist file or 
+    # Runs the discovery step.
+    set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${dispatch_script}")
+    if (ARG_USE_PRE_TEST_DISCOVERY)
+        set(dispatch_prefix "${CMAKE_CURRENT_BINARY_DIR}/discover_tests_${target}_")
+    else()
+        set(dispatch_prefix "${CMAKE_CURRENT_BINARY_DIR}/testlist_${target}_")
+
+        # Generate the testlist POST_BUILD
+        add_custom_command(
+            TARGET ${target} 
+            POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -P "${discovery_script}"
+            WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+            BYPRODUCTS "${testlist_file}"
+            VERBATIM
+        )
+    endif ()
+
+
+    # The discovery script is created at generation time to resolve $<TARGET_FILE:...>.
+    # This script will be run at build time (default) or test time (USE_PRE_TEST_DISCOVERY)
+    # to discover and either directly add or generate a testlist.cmake file.
+    set(discovery_script_content "\
+# Auto-generated test discovery script generated by fortuno_discover_tests(${target}).
+# $<CONFIG>
+cmake_policy(PUSH)
+cmake_policy(SET CMP0012 NEW) # Enable bools
+set(working_dir \"${ARG_WORKING_DIRECTORY}\")
+set(test_prefix \"${ARG_TEST_PREFIX}\")
+set(test_suffix \"${ARG_TEST_SUFFIX}\")
+set(test_executable \"$<TARGET_FILE:${target}>\")
+
+set(discover_command \"\${test_executable}\" --list ${ARG_TEST_PATTERN})
+
+# Query Fortuno executable for tests
+if (NOT EXISTS \"\${test_executable}\")
+    message(WARNING \"Could not find test executable: \${test_executable}. (Have you built the project?) Skipping test discovery.\")
+    cmake_policy(POP)
+    return()
+endif ()
+
+execute_process(
+    COMMAND \${discover_command}
+    OUTPUT_VARIABLE output_string
+    ERROR_VARIABLE error_string
+    RESULT_VARIABLE returncode
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if (returncode)
+    message(FATAL_ERROR \"Test discovery failed for ${target}, \${discover_command} returned \${returncode}.\\nOutput:\\n\${error_string}\")
+    cmake_policy(POP)
+    return()
+endif ()
+
+# Convert newlines to CMake semicolon list
+string(REGEX REPLACE \"(\\r\\n|\\n)\" \";\" testlist \"\${output_string}\")
+
+# Notify user if no tests found (e.q. due to bad pattern)
+if (testlist STREQUAL \"\")
+    message(WARNING \"No tests found for ${target} with pattern '${ARG_TEST_PATTERN}'.\")
+    cmake_policy(POP)
+    return()
+endif ()
+
+if (${ARG_USE_PRE_TEST_DISCOVERY})
+    message(STATUS \"Registering \${testlist} tests for ${target} (PRE_TEST discovery mode)...\")
+else()
+    file(WRITE \"${testlist_file}\" \"# Auto-generated test list for ${target} (POST_BUILD discovery mode)\\n\")
+endif ()
+
+
+# Actually discover each test
+foreach(test_name IN LISTS testlist)
+    set(ctest_name \"\${test_prefix}\${test_name}\${test_suffix}\")
+    set(test_command \${cross_emulator} \"\${test_executable}\" \"\${test_name}\")
+
+    # Either write tests to file, or add directly
+    if (${ARG_USE_PRE_TEST_DISCOVERY})
+        add_test(\"\${ctest_name}\" \${test_command})
+        set_tests_properties(\"\${ctest_name}\" PROPERTIES WORKING_DIRECTORY \"\${working_dir}\")
+    else()
+        file(APPEND \"${testlist_file}\" \"add_test(\\\"\${ctest_name}\\\" \${test_command})\\n\")
+        file(APPEND \"${testlist_file}\" \"set_tests_properties(\\\"\${ctest_name}\\\" PROPERTIES WORKING_DIRECTORY \\\"\${working_dir}\\\")\\n\")
+    endif ()
+endforeach()
+cmake_policy(POP)
+    ") # end discovery_script_content
+
+    set(dispatch_script_content "\
+# Test discovery dispatch script generated by fortuno_discover_tests(${target}).
+cmake_policy(PUSH)
+cmake_policy(SET CMP0012 NEW)
+
+if (DEFINED CTEST_CONFIGURATION_TYPE AND CTEST_CONFIGURATION_TYPE)
+    set(script_path \"${dispatch_prefix}\${CTEST_CONFIGURATION_TYPE}.cmake\")
+else()
+    set(script_path \"${dispatch_prefix}${CMAKE_BUILD_TYPE}.cmake\")
+endif ()
+
+if (EXISTS \"\${script_path}\")
+    include(\"\${script_path}\")
+endif ()
+cmake_policy(POP)
+") # end dispatch_script_content
+
+    file(WRITE "${dispatch_script}" "${dispatch_script_content}")
+    file(GENERATE OUTPUT "${discovery_script}" CONTENT "${discovery_script_content}")
+
+
+endfunction()
+

cmake/FortunoConfig.cmake.in

@@ -1,3 +1,5 @@
 @PACKAGE_INIT@
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
+
 include(${CMAKE_CURRENT_LIST_DIR}/FortunoTargets.cmake)

src/fortuno/argumentparser.f90

@@ -160,7 +160,19 @@ subroutine argument_parser_parse_args(this, argumentvalues, logger, exitcode)
           cycle
         end if
         if (.not. optionsallowed .or. arg(1:1) /= "-") then
-          posargs = [posargs, string_item(arg)]
+          ! Workaround:gfortran:14.1 (bug 116679)
+          ! Omit array expression to avoid memory leak
+          ! {-
+          ! posargs = [posargs, string_item(arg)]
+          ! -}{+
+          block
+            type(string_item), allocatable :: tmp(:)
+            allocate(tmp(size(posargs) + 1))
+            if (size(posargs) > 0) tmp(1:size(posargs)) = posargs
+            tmp(size(posargs) + 1) = string_item(arg)
+            call move_alloc(tmp, posargs)
+          end block
+          ! +}
           cycle
         end if
         islong = arg(1 : min(len(arg), 2)) == "--"
@@ -410,7 +422,7 @@ subroutine print_argument_help_(logger, argument, helpmsg, linelength)
     character(*), intent(in) :: argument, helpmsg
     integer, intent(in) :: linelength
 
-    integer, parameter :: offset = 25
+    integer, parameter :: offset = 29
     character(20) :: formatstr
     character(linelength) :: buffer
     integer :: maxwidth, curpos, seppos
@@ -447,4 +459,4 @@ subroutine print_argument_help_(logger, argument, helpmsg, linelength)
 
   end subroutine print_argument_help_
 
-end module fortuno_argumentparser
+end module fortuno_argumentparser

src/fortuno/cmdapp.f90

@@ -9,6 +9,7 @@ module fortuno_cmdapp
   use fortuno_basetypes, only : error_info, test_list
   use fortuno_utils, only : string_item
   use fortuno_testdriver, only : test_driver, test_selection
+  use fortuno_testinfo, only : teststatus
   use fortuno_testlogger, only : test_logger
   implicit none
 
@@ -89,6 +90,7 @@ subroutine cmd_app_register_tests(this, testitems, exitcode)
     type(test_selection), allocatable :: selections(:)
     type(string_item), allocatable :: selectors(:), testnames(:)
     type(error_info), allocatable :: error
+    logical :: strict_matching_success
     integer :: itest
 
     exitcode = -1
@@ -101,10 +103,17 @@ subroutine cmd_app_register_tests(this, testitems, exitcode)
       end if
       call get_selections(selectors, selections)
     end if
-    call this%driver%register_tests(testitems, selections=selections)
+    call this%driver%register_tests(testitems, strict_matching_success, selections=selections)
+
+    if (.not. strict_matching_success .or. this%argvals%has("ignore-no-match") ) then
+      call this%logger%log_error("Error: One or more test selection arguments had no effect.&
+          & Use --ignore-no-match to disable this error.")
+      exitcode = 1
+    end if
+
+    call this%driver%get_test_names(testnames)
 
     if (this%argvals%has("list")) then
-      call this%driver%get_test_names(testnames)
       do itest = 1, size(testnames)
         call this%logger%log_message(testnames(itest)%value)
       end do
@@ -125,10 +134,11 @@ subroutine cmd_app_run_tests(this, exitcode)
     integer, intent(out) :: exitcode
 
     call this%driver%run_tests(this%logger)
-    if (this%driver%driveresult%successful) then
-      exitcode = 0
-    else
+
+    if (.not. this%driver%driveresult%successful) then
       exitcode = 1
+    else
+      exitcode = 0
     end if
 
   end subroutine cmd_app_run_tests
@@ -175,17 +185,24 @@ function default_argument_defs() result(argdefs)
     !     & argument_def("list", argtypes%bool, shortopt="l", longopt="list",&
     !     & helpmsg="show list of tests to run and exit"),&
     !     & &
+    !     & argument_def("ignore-no-match", argtypes%bool, &
+    !     & longopt="ignore-no-match",&
+    !     & helpmsg="Allow test selection arguments with no effect."),&
+    !     & &
     !     & argument_def("tests", argtypes%stringlist,&
     !     & helpmsg="list of tests and suites to include or to exclude when prefixed with '~' (e.g.&
     !     & 'somesuite ~somesuite/avoidedtest' would run all tests except 'avoidedtest' in the test&
     !     & suite 'somesuite')")&
     !     & &
     !     & ]
     ! -}{+
-    allocate(argdefs(2))
-    argdefs(1) =  argument_def("list", argtypes%bool, shortopt="l", longopt="list",&
+    allocate(argdefs(3))
+    argdefs(1) = argument_def("list", argtypes%bool, shortopt="l", longopt="list",&
         & helpmsg="show list of tests to run and exit")
-    argdefs(2) = argument_def("tests", argtypes%stringlist,&
+    argdefs(2) = argument_def("disable-strict-matching", argtypes%bool, &
+        & longopt="ignore-no-match",&
+        & helpmsg="Allow test selection arguments with no effect.")
+    argdefs(3) = argument_def("tests", argtypes%stringlist,&
         & helpmsg="list of tests and suites to include or to exclude when prefixed with '~' (e.g.&
         & 'somesuite ~somesuite/avoidedtest' would run all tests except 'avoidedtest' in the test&
         & suite 'somesuite')")

src/fortuno/consolelogger.f90

@@ -425,4 +425,4 @@ subroutine log_success_(successful)
 
   end subroutine log_success_
 
-end module fortuno_consolelogger
+end module fortuno_consolelogger