diff --git a/CMakeLists.txt b/CMakeLists.txt index d457f14..ac90c2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,8 +118,14 @@ if (FORTUNO_INSTALL) ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfig.cmake INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Fortuno ) + configure_file( + cmake/Fortuno.cmake + ${CMAKE_CURRENT_BINARY_DIR}/Fortuno.cmake + COPYONLY + ) install( FILES + ${CMAKE_CURRENT_BINARY_DIR}/Fortuno.cmake ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfigVersion.cmake ${CMAKE_CURRENT_BINARY_DIR}/FortunoConfig.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Fortuno @@ -143,6 +149,9 @@ endif () # Make project available for FetchContent if (NOT PROJECT_IS_TOP_LEVEL) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE) + # Propagate variables if (CMAKE_VERSION VERSION_LESS 3.25) # TODO: Remove when required minimum cmake version is >= 3.25 diff --git a/README.rst b/README.rst index f20f663..00c05d3 100644 --- a/README.rst +++ b/README.rst @@ -4,8 +4,8 @@ Fortuno – flextensible unit testing framework for Fortran **Fortuno** (Fortran Unit Testing Objects) is a flexible & extensible, object-oriented unit testing framework designed for the Fortran programming -language. It emphasizes ease of use by minimizing boiler plate code when writing -tests while prioratizing modularity and extensibility. Fortuno provides the +language. It emphasizes ease of use by minimizing boilerplate code when writing +tests while prioritizing modularity and extensibility. Fortuno provides the essential building blocks to help developers create customized unit testing solutions. @@ -336,12 +336,17 @@ your chosen build system: * **CMake**: In your ``CMakeLists.txt`` file, declare an executable ``testapp`` using ``testapp.f90`` as the source file and add ``Fortuno::fortuno_serial`` as a dependency. Be sure to also link your library (e.g. ``mylib``). - Additionally, register the executable as a test, so that it can be executed - with ``ctest``:: + Finally, use ``fortuno_discover_tests()`` to register the tests, so that they + can be executed with ``ctest``:: add_executable(testapp testapp.f90) target_link_libraries(testapp PRIVATE mylib Fortuno::fortuno_serial) - add_test(NAME factorial COMMAND testapp) + fortuno_discover_tests(testapp) + + *Note*: Alternatively, you can register the test executable manually with + ``add_test(NAME factorial COMMAND testapp)``. However, + ``fortuno_discover_tests()`` is recommended as it automatically discovers and + registers all tests in the executable. *Note*: If you are using the MPI or coarray interface, replace ``Fortuno::fortuno_serial`` with ``Fortuno::fortuno_mpi`` or diff --git a/cmake/Fortuno.cmake b/cmake/Fortuno.cmake new file mode 100644 index 0000000..8915450 --- /dev/null +++ b/cmake/Fortuno.cmake @@ -0,0 +1,146 @@ +include_guard(GLOBAL) +# Usage: +# fortuno_discover_tests(target, +# [USE_PRE_TEST_DISCOVERY] # (Optional) flag to use pre-test discovery mode (default is post-build) +# [TEST_PREFIX prefix] # (Optional) prefix for test names +# [TEST_SUFFIX suffix] # (Optional) suffix for test names +# [TEST_PATTERN pattern] # (Optional) pattern to filter tests (default is none, i.e. all tests) +# [WORKING_DIRECTORY dir] # (Optional) working directory for tests (default is CMAKE_CURRENT_BINARY_DIR) +# Description: +# Query the Fortuno test executable for its list of tests and individually register them with CTest. +# Querying is performed either at build time (default POST_BUILD mode) or at test time (PRE_TEST_DISCOVERY mode). +# In the former case, an additional testlist.cmake file is generated after building the target to be included by CTest. +# In the latter case, the test discovery is performed anew each time tests are run. +function(fortuno_discover_tests target) + if (NOT TARGET ${target}) + message(FATAL_ERROR "fortuno_discover_tests: target '${target}' does not exist") + endif () + + # Parse arguments and apply defaults + cmake_parse_arguments(ARG "USE_PRE_TEST_DISCOVERY" "TEST_PREFIX;TEST_SUFFIX;TEST_PATTERN;WORKING_DIRECTORY" "" ${ARGN}) + + if (ARG_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "fortuno_discover_tests: Unknown arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif () + + if (NOT DEFINED ARG_WORKING_DIRECTORY) + set(ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif () + + + set(discovery_script "${CMAKE_CURRENT_BINARY_DIR}/discover_tests_${target}_$.cmake") + set(dispatch_script "${CMAKE_CURRENT_BINARY_DIR}/dispatch_discover_${target}.cmake") + set(testlist_file "${CMAKE_CURRENT_BINARY_DIR}/testlist_${target}_$.cmake") + + # Point ctest to the dispatch script which either loads the testlist file or + # Runs the discovery step. + set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${dispatch_script}") + if (ARG_USE_PRE_TEST_DISCOVERY) + set(dispatch_prefix "${CMAKE_CURRENT_BINARY_DIR}/discover_tests_${target}_") + else() + set(dispatch_prefix "${CMAKE_CURRENT_BINARY_DIR}/testlist_${target}_") + + # Generate the testlist POST_BUILD + add_custom_command( + TARGET ${target} + POST_BUILD + COMMAND ${CMAKE_COMMAND} -P "${discovery_script}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + BYPRODUCTS "${testlist_file}" + VERBATIM + ) + endif () + + + # The discovery script is created at generation time to resolve $. + # This script will be run at build time (default) or test time (USE_PRE_TEST_DISCOVERY) + # to discover and either directly add or generate a testlist.cmake file. + set(discovery_script_content "\ +# Auto-generated test discovery script generated by fortuno_discover_tests(${target}). +# $ +cmake_policy(PUSH) +cmake_policy(SET CMP0012 NEW) # Enable bools +set(working_dir \"${ARG_WORKING_DIRECTORY}\") +set(test_prefix \"${ARG_TEST_PREFIX}\") +set(test_suffix \"${ARG_TEST_SUFFIX}\") +set(test_executable \"$\") + +set(discover_command \"\${test_executable}\" --list ${ARG_TEST_PATTERN}) + +# Query Fortuno executable for tests +if (NOT EXISTS \"\${test_executable}\") + message(WARNING \"Could not find test executable: \${test_executable}. (Have you built the project?) Skipping test discovery.\") + cmake_policy(POP) + return() +endif () + +execute_process( + COMMAND \${discover_command} + OUTPUT_VARIABLE output_string + ERROR_VARIABLE error_string + RESULT_VARIABLE returncode + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if (returncode) + message(FATAL_ERROR \"Test discovery failed for ${target}, \${discover_command} returned \${returncode}.\\nOutput:\\n\${error_string}\") + cmake_policy(POP) + return() +endif () + +# Convert newlines to CMake semicolon list +string(REGEX REPLACE \"(\\r\\n|\\n)\" \";\" testlist \"\${output_string}\") + +# Notify user if no tests found (e.q. due to bad pattern) +if (testlist STREQUAL \"\") + message(WARNING \"No tests found for ${target} with pattern '${ARG_TEST_PATTERN}'.\") + cmake_policy(POP) + return() +endif () + +if (${ARG_USE_PRE_TEST_DISCOVERY}) + message(STATUS \"Registering \${testlist} tests for ${target} (PRE_TEST discovery mode)...\") +else() + file(WRITE \"${testlist_file}\" \"# Auto-generated test list for ${target} (POST_BUILD discovery mode)\\n\") +endif () + + +# Actually discover each test +foreach(test_name IN LISTS testlist) + set(ctest_name \"\${test_prefix}\${test_name}\${test_suffix}\") + set(test_command \${cross_emulator} \"\${test_executable}\" \"\${test_name}\") + + # Either write tests to file, or add directly + if (${ARG_USE_PRE_TEST_DISCOVERY}) + add_test(\"\${ctest_name}\" \${test_command}) + set_tests_properties(\"\${ctest_name}\" PROPERTIES WORKING_DIRECTORY \"\${working_dir}\") + else() + file(APPEND \"${testlist_file}\" \"add_test(\\\"\${ctest_name}\\\" \${test_command})\\n\") + file(APPEND \"${testlist_file}\" \"set_tests_properties(\\\"\${ctest_name}\\\" PROPERTIES WORKING_DIRECTORY \\\"\${working_dir}\\\")\\n\") + endif () +endforeach() +cmake_policy(POP) + ") # end discovery_script_content + + set(dispatch_script_content "\ +# Test discovery dispatch script generated by fortuno_discover_tests(${target}). +cmake_policy(PUSH) +cmake_policy(SET CMP0012 NEW) + +if (DEFINED CTEST_CONFIGURATION_TYPE AND CTEST_CONFIGURATION_TYPE) + set(script_path \"${dispatch_prefix}\${CTEST_CONFIGURATION_TYPE}.cmake\") +else() + set(script_path \"${dispatch_prefix}${CMAKE_BUILD_TYPE}.cmake\") +endif () + +if (EXISTS \"\${script_path}\") + include(\"\${script_path}\") +endif () +cmake_policy(POP) +") # end dispatch_script_content + + file(WRITE "${dispatch_script}" "${dispatch_script_content}") + file(GENERATE OUTPUT "${discovery_script}" CONTENT "${discovery_script_content}") + + +endfunction() + diff --git a/cmake/FortunoConfig.cmake.in b/cmake/FortunoConfig.cmake.in index 661c17c..7f4253f 100644 --- a/cmake/FortunoConfig.cmake.in +++ b/cmake/FortunoConfig.cmake.in @@ -1,3 +1,5 @@ @PACKAGE_INIT@ +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + include(${CMAKE_CURRENT_LIST_DIR}/FortunoTargets.cmake) diff --git a/src/fortuno/argumentparser.f90 b/src/fortuno/argumentparser.f90 index 1f76ed9..d2716cc 100644 --- a/src/fortuno/argumentparser.f90 +++ b/src/fortuno/argumentparser.f90 @@ -160,7 +160,19 @@ subroutine argument_parser_parse_args(this, argumentvalues, logger, exitcode) cycle end if if (.not. optionsallowed .or. arg(1:1) /= "-") then - posargs = [posargs, string_item(arg)] + ! Workaround:gfortran:14.1 (bug 116679) + ! Omit array expression to avoid memory leak + ! {- + ! posargs = [posargs, string_item(arg)] + ! -}{+ + block + type(string_item), allocatable :: tmp(:) + allocate(tmp(size(posargs) + 1)) + if (size(posargs) > 0) tmp(1:size(posargs)) = posargs + tmp(size(posargs) + 1) = string_item(arg) + call move_alloc(tmp, posargs) + end block + ! +} cycle end if islong = arg(1 : min(len(arg), 2)) == "--" @@ -410,7 +422,7 @@ subroutine print_argument_help_(logger, argument, helpmsg, linelength) character(*), intent(in) :: argument, helpmsg integer, intent(in) :: linelength - integer, parameter :: offset = 25 + integer, parameter :: offset = 29 character(20) :: formatstr character(linelength) :: buffer integer :: maxwidth, curpos, seppos @@ -447,4 +459,4 @@ subroutine print_argument_help_(logger, argument, helpmsg, linelength) end subroutine print_argument_help_ -end module fortuno_argumentparser \ No newline at end of file +end module fortuno_argumentparser diff --git a/src/fortuno/cmdapp.f90 b/src/fortuno/cmdapp.f90 index 7aef70d..7c84b92 100644 --- a/src/fortuno/cmdapp.f90 +++ b/src/fortuno/cmdapp.f90 @@ -9,6 +9,7 @@ module fortuno_cmdapp use fortuno_basetypes, only : error_info, test_list use fortuno_utils, only : string_item use fortuno_testdriver, only : test_driver, test_selection + use fortuno_testinfo, only : teststatus use fortuno_testlogger, only : test_logger implicit none @@ -89,6 +90,7 @@ subroutine cmd_app_register_tests(this, testitems, exitcode) type(test_selection), allocatable :: selections(:) type(string_item), allocatable :: selectors(:), testnames(:) type(error_info), allocatable :: error + logical :: strict_matching_success integer :: itest exitcode = -1 @@ -101,10 +103,17 @@ subroutine cmd_app_register_tests(this, testitems, exitcode) end if call get_selections(selectors, selections) end if - call this%driver%register_tests(testitems, selections=selections) + call this%driver%register_tests(testitems, strict_matching_success, selections=selections) + + if (.not. strict_matching_success .or. this%argvals%has("ignore-no-match") ) then + call this%logger%log_error("Error: One or more test selection arguments had no effect.& + & Use --ignore-no-match to disable this error.") + exitcode = 1 + end if + + call this%driver%get_test_names(testnames) if (this%argvals%has("list")) then - call this%driver%get_test_names(testnames) do itest = 1, size(testnames) call this%logger%log_message(testnames(itest)%value) end do @@ -125,10 +134,11 @@ subroutine cmd_app_run_tests(this, exitcode) integer, intent(out) :: exitcode call this%driver%run_tests(this%logger) - if (this%driver%driveresult%successful) then - exitcode = 0 - else + + if (.not. this%driver%driveresult%successful) then exitcode = 1 + else + exitcode = 0 end if end subroutine cmd_app_run_tests @@ -175,6 +185,10 @@ function default_argument_defs() result(argdefs) ! & argument_def("list", argtypes%bool, shortopt="l", longopt="list",& ! & helpmsg="show list of tests to run and exit"),& ! & & + ! & argument_def("ignore-no-match", argtypes%bool, & + ! & longopt="ignore-no-match",& + ! & helpmsg="Allow test selection arguments with no effect."),& + ! & & ! & argument_def("tests", argtypes%stringlist,& ! & helpmsg="list of tests and suites to include or to exclude when prefixed with '~' (e.g.& ! & 'somesuite ~somesuite/avoidedtest' would run all tests except 'avoidedtest' in the test& @@ -182,10 +196,13 @@ function default_argument_defs() result(argdefs) ! & & ! & ] ! -}{+ - allocate(argdefs(2)) - argdefs(1) = argument_def("list", argtypes%bool, shortopt="l", longopt="list",& + allocate(argdefs(3)) + argdefs(1) = argument_def("list", argtypes%bool, shortopt="l", longopt="list",& & helpmsg="show list of tests to run and exit") - argdefs(2) = argument_def("tests", argtypes%stringlist,& + argdefs(2) = argument_def("disable-strict-matching", argtypes%bool, & + & longopt="ignore-no-match",& + & helpmsg="Allow test selection arguments with no effect.") + argdefs(3) = argument_def("tests", argtypes%stringlist,& & helpmsg="list of tests and suites to include or to exclude when prefixed with '~' (e.g.& & 'somesuite ~somesuite/avoidedtest' would run all tests except 'avoidedtest' in the test& & suite 'somesuite')") diff --git a/src/fortuno/consolelogger.f90 b/src/fortuno/consolelogger.f90 index 75ad5d5..5f172e9 100644 --- a/src/fortuno/consolelogger.f90 +++ b/src/fortuno/consolelogger.f90 @@ -425,4 +425,4 @@ subroutine log_success_(successful) end subroutine log_success_ -end module fortuno_consolelogger \ No newline at end of file +end module fortuno_consolelogger diff --git a/src/fortuno/testdriver.f90 b/src/fortuno/testdriver.f90 index 8911bfd..148600a 100644 --- a/src/fortuno/testdriver.f90 +++ b/src/fortuno/testdriver.f90 @@ -192,7 +192,7 @@ end subroutine final_test_driver !> Registers tests to consider - subroutine test_driver_register_tests(this, testlist, selections) + subroutine test_driver_register_tests(this, testlist, strict_matching_success, selections) !> Instance class(test_driver), intent(inout) :: this @@ -203,13 +203,16 @@ subroutine test_driver_register_tests(this, testlist, selections) !> Selection rule to constrain the testing only to a subset of the test items type(test_selection), optional, intent(in) :: selections(:) + !> True if all selections were matched successfully, false otherwise + logical, intent(out) :: strict_matching_success + this%testlist = testlist call init_test_data_container(this%suitedatacont, 100) call init_test_data_container(this%testdatacont, 5000) call build_test_data_(this%testlist, "", [integer ::], [integer ::], this%testdatacont,& & this%suitedatacont) call get_selected_suites_and_tests_(this%suitedatacont, this%testdatacont, this%suiteselection,& - & this%testselection, selections) + & this%testselection, strict_matching_success, selections) end subroutine test_driver_register_tests @@ -572,17 +575,20 @@ end subroutine set_repr_name_ !! Returns indices of selected suites and tests. subroutine get_selected_suites_and_tests_(suitedatacont, testdatacont, suiteselection,& - & testselection, selections) + & testselection, strict_matching_success, selections) type(test_data_container), intent(in) :: suitedatacont, testdatacont type(reversible_mapping), intent(out) :: suiteselection, testselection type(test_selection), optional, intent(in) :: selections(:) + logical, intent(out) :: strict_matching_success logical, allocatable :: testmask(:), suitemask(:) - logical :: hasselection, selected, isincluded + logical :: hasselection, selected, isincluded, found_match integer :: iselect, itest integer :: selectnamelen integer :: ii + strict_matching_success = .true. + hasselection = present(selections) if (hasselection) hasselection = size(selections) > 0 if (.not. hasselection) then @@ -597,6 +603,7 @@ subroutine get_selected_suites_and_tests_(suitedatacont, testdatacont, suitesele ! If first option is an exclusion, include all tests by default otherwise exclude them. testmask(:) = selections(1)%selectiontype == "-" do iselect = 1, size(selections) + found_match = .false. associate(selection => selections(iselect)) isincluded = selection%selectiontype == "+" selectnamelen = len(selection%name) @@ -610,9 +617,16 @@ subroutine get_selected_suites_and_tests_(suitedatacont, testdatacont, suitesele else selected = .false. end if - if (selected) testmask(itest) = isincluded + + if (selected) then + testmask(itest) = isincluded + found_match = .true. + end if + end associate end do + + if (.not. found_match) strict_matching_success = .false. end associate end do diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 488a00c..24b42a2 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -15,4 +15,8 @@ target_sources( testapp.f90 ) target_link_libraries(fortuno_test_unit_testapp PRIVATE Fortuno::fortuno_serial) -add_test(NAME unit COMMAND testapp) + + +include(${PROJECT_SOURCE_DIR}/cmake/Fortuno.cmake) + +fortuno_discover_tests(fortuno_test_unit_testapp USE_PRE_TEST_DISCOVERY)