Skip to content

feat(test): Add skip ability to the test cases at runtime #609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davide221 merged 4 commits into from
Aug 17, 2026
Merged

feat(test): Add skip ability to the test cases at runtime #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcbc2dc
convert smoke tests to ctest
howard0su Aug 13, 2026
9c929c9
Migrate standalone tests to CppUnitTestFramework
howard0su Aug 13, 2026
126ad13
Handle unavailable test runtimes as skips
howard0su Aug 15, 2026
7ac6790
Merge branch 'main' into test_skip
davide221 Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 127 additions & 52 deletions server/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,7 +808,6 @@ if(DFLASH27B_TESTS)
if(WIN32)
target_link_libraries(test_platform_compat PRIVATE ws2_32)
endif()
add_test(NAME platform_compat COMMAND test_platform_compat)
endif()

if(DFLASH27B_GPU_BACKEND STREQUAL "hip"
Expand All @@ -832,7 +831,6 @@ if(DFLASH27B_TESTS)
${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include)
target_link_libraries(test_cuda_comm_api PRIVATE
ggml-cuda ggml ggml-base)
add_test(NAME cuda_comm_api COMMAND test_cuda_comm_api)
endif()

if(DFLASH27B_ENABLE_MIXED_CUDA_HIP)
Expand Down Expand Up @@ -902,7 +900,6 @@ if(DFLASH27B_TESTS)
add_executable(test_qwen35_tensor_parallel test/test_qwen35_tensor_parallel.cpp)
target_include_directories(test_qwen35_tensor_parallel PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
target_link_libraries(test_qwen35_tensor_parallel PRIVATE dflash_common)
add_test(NAME qwen35_tensor_parallel COMMAND test_qwen35_tensor_parallel)

if(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND _dflash_cuda_min_sm GREATER_EQUAL 80 AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_flashprefill_kernels.cpp")
add_executable(test_flashprefill_kernels test/test_flashprefill_kernels.cpp)
Expand Down Expand Up @@ -1065,7 +1062,6 @@ if(DFLASH27B_TESTS)
find_package(CUDAToolkit REQUIRED)
target_link_libraries(test_ggml_rmsnorm_batch PRIVATE CUDA::cudart)
endif()
add_test(NAME ggml_rmsnorm_batch COMMAND test_ggml_rmsnorm_batch)
endif()
# POSIX-only fixture: it builds its temp model/sidecar stand-ins with symlink()/getpid()
# under /tmp and probes them with access(R_OK). Porting that to Win32 would mean
Expand Down Expand Up @@ -1214,7 +1210,6 @@ if(DFLASH27B_TESTS)
else()
target_link_libraries(test_ds4_mix_registry_teardown PRIVATE hip::host)
endif()
add_test(NAME ds4_mix_registry_teardown COMMAND test_ds4_mix_registry_teardown)
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ds4_budget_hook.cpp")
# Thinking-budget force-close rule. Header-only logic, so unlike the other ds4 tests
Expand All @@ -1239,7 +1234,6 @@ if(DFLASH27B_TESTS)
else()
target_link_libraries(test_ds4_dmix_entry_validation PRIVATE hip::host)
endif()
add_test(NAME ds4_dmix_entry_validation COMMAND test_ds4_dmix_entry_validation)
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ds4_dspark_load.cpp")
add_executable(test_ds4_dspark_load test/test_ds4_dspark_load.cpp)
Expand All @@ -1257,11 +1251,6 @@ if(DFLASH27B_TESTS)
target_link_libraries(test_ds4_dspark_load PRIVATE hip::host)
endif()
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_draft.cpp")
add_executable(smoke_load_draft test/smoke_load_draft.cpp)
target_include_directories(smoke_load_draft PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
target_link_libraries(smoke_load_draft PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET})
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/spike_thin_copy.cpp")
add_executable(spike_thin_copy test/spike_thin_copy.cpp)
target_include_directories(spike_thin_copy PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
Expand All @@ -1282,16 +1271,6 @@ if(DFLASH27B_TESTS)
target_include_directories(test_vs_oracle PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
target_link_libraries(test_vs_oracle PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET})
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_target.cpp")
add_executable(smoke_load_target test/smoke_load_target.cpp)
target_include_directories(smoke_load_target PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
target_link_libraries(smoke_load_target PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET})
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_target_laguna.cpp")
add_executable(smoke_load_target_laguna test/smoke_load_target_laguna.cpp)
target_include_directories(smoke_load_target_laguna PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
target_link_libraries(smoke_load_target_laguna PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET})
endif()
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_laguna_forward.cpp")
add_executable(smoke_laguna_forward test/smoke_laguna_forward.cpp)
target_include_directories(smoke_laguna_forward PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
Expand Down Expand Up @@ -1479,6 +1458,7 @@ if(DFLASH27B_TESTS)
test/test_drafter_tail_capture_guard.cpp
test/test_drafter_warm_path_regression.cpp
test/test_qwen3_buffer_plan.cpp
test/test_model_test_paths.cpp
test/test_gguf_mmap.cpp
test/test_kv_quant.cpp
test/test_kvflash_placement.cpp
Expand Down Expand Up @@ -1557,6 +1537,69 @@ if(DFLASH27B_TESTS)
list(APPEND _raw_unit_test_targets test_server_unit)
endif()

if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
DFLASH27B_GPU_BACKEND STREQUAL "hip")
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_draft.cpp"
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_target.cpp"
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_load_target_laguna.cpp"
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_paged_attention.cpp")
add_executable(test_model_smoke
test/test_unit_main.cpp
test/smoke_load_draft.cpp
test/smoke_load_target.cpp
test/smoke_load_target_laguna.cpp
test/test_paged_attention.cpp)
target_include_directories(test_model_smoke PRIVATE
${DFLASH27B_SRC_INCLUDE_DIRS}
${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src)
if(DFLASH27B_GPU_BACKEND STREQUAL "hip")
target_compile_definitions(test_model_smoke PRIVATE
DFLASH27B_BACKEND_HIP=1
GGML_USE_HIP)
else()
target_compile_definitions(test_model_smoke PRIVATE
DFLASH27B_BACKEND_CUDA=1
DFLASH27B_CUDA_MIN_SM=${_dflash_cuda_min_sm})
endif()
target_link_libraries(test_model_smoke PRIVATE
dflash_common
ggml
ggml-base
${DFLASH27B_GGML_BACKEND_TARGET})
if(DFLASH27B_GPU_BACKEND STREQUAL "cuda")
find_package(CUDAToolkit REQUIRED)
target_link_libraries(test_model_smoke PRIVATE CUDA::cudart)
else()
target_link_libraries(test_model_smoke PRIVATE hip::host)
endif()
if(CMAKE_CROSSCOMPILING)
add_test(NAME model_smoke COMMAND test_model_smoke)
Comment thread
howard0su marked this conversation as resolved.
set_tests_properties(model_smoke PROPERTIES SKIP_RETURN_CODE 77)
else()
set(_model_smoke_ctest_generated
"${CMAKE_CURRENT_BINARY_DIR}/test_model_smoke_discovered_tests.cmake")
set(_model_smoke_ctest_include
"${CMAKE_CURRENT_BINARY_DIR}/test_model_smoke_discovered_tests_include.cmake")
file(GENERATE OUTPUT "${_model_smoke_ctest_include}"
CONTENT "include([==[${_model_smoke_ctest_generated}]==] OPTIONAL)\n")
set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES
"${_model_smoke_ctest_include}")
add_custom_command(TARGET test_model_smoke POST_BUILD
COMMAND ${CMAKE_COMMAND}
-DTEST_EXECUTABLE=$<TARGET_FILE:test_model_smoke>
-DTEST_WORKING_DIR=${CMAKE_CURRENT_BINARY_DIR}
-DCTEST_FILE=${_model_smoke_ctest_generated}
-DTEST_PREFIX=test_model_smoke.
-P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DiscoverCppUnitTests.cmake
VERBATIM
COMMENT "Discovering CppUnitTestFramework tests for test_model_smoke")
unset(_model_smoke_ctest_generated)
unset(_model_smoke_ctest_include)
endif()
list(APPEND _raw_unit_test_targets test_model_smoke)
endif()

# Feature/architecture gate tests. check_feature_compatibility(),
# collect_feature_warnings() and the capability table are pure functions,
# so this target deliberately compiles only feature_gate.cpp and
Expand All @@ -1581,12 +1624,69 @@ if(DFLASH27B_TESTS)
target_compile_definitions(test_feature_gate PRIVATE
DFLASH27B_BACKEND_CUDA=1)
endif()
add_test(NAME feature_gate COMMAND test_feature_gate)
endif()

function(dflash_discover_cppunit_tests target)
if(CMAKE_CROSSCOMPILING)
add_test(NAME "${target}" COMMAND ${target})
set_tests_properties("${target}" PROPERTIES SKIP_RETURN_CODE 77)
return()
endif()

set(_ctest_generated
"${CMAKE_CURRENT_BINARY_DIR}/${target}_discovered_tests.cmake")
set(_ctest_include
"${CMAKE_CURRENT_BINARY_DIR}/${target}_discovered_tests_include.cmake")
file(GENERATE OUTPUT "${_ctest_include}"
CONTENT "include([==[${_ctest_generated}]==] OPTIONAL)\n")
set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES
"${_ctest_include}")
add_custom_command(TARGET ${target} POST_BUILD
COMMAND ${CMAKE_COMMAND}
-DTEST_EXECUTABLE=$<TARGET_FILE:${target}>
-DTEST_WORKING_DIR=${CMAKE_CURRENT_BINARY_DIR}
-DCTEST_FILE=${_ctest_generated}
-DTEST_PREFIX=${target}.
-P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DiscoverCppUnitTests.cmake
VERBATIM
COMMENT "Discovering CppUnitTestFramework tests for ${target}")
endfunction()

set(_cppunit_test_targets
test_server_unit
test_model_smoke)
set(_new_cppunit_test_targets
test_platform_compat
test_feature_gate
test_qwen35_split_tree_guard
test_recurrent_snapshot
test_ds4_mix_registry_teardown
test_ds4_dmix_entry_validation
test_ggml_rmsnorm_batch
test_qwen35_tensor_parallel
test_deepseek4_hc_cuda
test_flashprefill_kernels
test_cuda_comm_api
test_mmq_streamk_iq4_xs
test_rms_norm_hip
test_rocmfp3_mix_registry
test_rocmfp_mix_slice_matvec
test_rocmfp_mix_gateup_glu
test_rocmfp_mix_glu_fusable)
foreach(_cppunit_target IN LISTS _new_cppunit_test_targets)
if(TARGET ${_cppunit_target})
target_sources(${_cppunit_target} PRIVATE test/test_unit_main.cpp)
dflash_discover_cppunit_tests(${_cppunit_target})

@cubic-dev-ai cubic-dev-ai Bot Aug 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Converting the listed targets to discovery drops the stable CTest names (platform_compat, cuda_comm_api, qwen35_tensor_parallel, ggml_rmsnorm_batch, feature_gate, ...) in favor of test_<target>.<method> without adding an alias, contradicting this same file's comment about preserving established names for CI filters/downstream scripts. If any filter references those names, tests silently stop matching.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At server/CMakeLists.txt, line 1675:

<comment>Converting the listed targets to discovery drops the stable CTest names (`platform_compat`, `cuda_comm_api`, `qwen35_tensor_parallel`, `ggml_rmsnorm_batch`, `feature_gate`, ...) in favor of `test_<target>.<method>` without adding an alias, contradicting this same file's comment about preserving established names for CI filters/downstream scripts. If any filter references those names, tests silently stop matching.</comment>

<file context>
@@ -1578,12 +1620,69 @@ if(DFLASH27B_TESTS)
+    foreach(_cppunit_target IN LISTS _new_cppunit_test_targets)
+        if(TARGET ${_cppunit_target})
+            target_sources(${_cppunit_target} PRIVATE test/test_unit_main.cpp)
+            dflash_discover_cppunit_tests(${_cppunit_target})
+            list(APPEND _cppunit_test_targets ${_cppunit_target})
+            list(APPEND _raw_unit_test_targets ${_cppunit_target})
</file context>

list(APPEND _cppunit_test_targets ${_cppunit_target})
list(APPEND _raw_unit_test_targets ${_cppunit_target})
endif()
endforeach()
unset(_new_cppunit_test_targets)

if(_raw_unit_test_targets)
foreach(_unit_target IN LISTS _raw_unit_test_targets)
if(NOT _unit_target STREQUAL "test_server_unit")
list(FIND _cppunit_test_targets "${_unit_target}" _cppunit_target_index)
if(_cppunit_target_index EQUAL -1)
# Preserve the established CTest names used by CI filters and
# downstream scripts even though dependency tracking uses the
# executable target names.
Expand Down Expand Up @@ -1615,9 +1715,7 @@ if(DFLASH27B_TESTS)
set(_unit_ctest_name paged_kv_pool)
endif()
add_test(NAME "${_unit_ctest_name}" COMMAND ${_unit_target})
if(_unit_target STREQUAL "test_deepseek4_mmid_grouped_cuda")
set_tests_properties("${_unit_ctest_name}" PROPERTIES SKIP_RETURN_CODE 77)
endif()
set_tests_properties("${_unit_ctest_name}" PROPERTIES SKIP_RETURN_CODE 77)
if(_unit_target STREQUAL "test_rocmfp3_mix_registry")
# Its leak assertion reads GLOBAL free VRAM (cudaMemGetInfo) before and
# after 4000 register/unregister cycles, so any other test allocating on
Expand All @@ -1631,7 +1729,9 @@ if(DFLASH27B_TESTS)
endif()
endforeach()
unset(_unit_ctest_name)
unset(_cppunit_target_index)
endif()
unset(_cppunit_test_targets)

# 'make check' — build every registered unit-test target, including the
# lightweight feature gate added on main, then run the complete CTest set.
Expand Down Expand Up @@ -1662,18 +1762,16 @@ if(DFLASH27B_TESTS)
# CUDA::cudart so the toolkit headers are on the compile line (same as
# test_dflash historically had alone).
set(_dflash_internal_h_cuda_tests
smoke_load_draft
smoke_draft_graph
test_vs_oracle
smoke_load_target
smoke_load_target_laguna
smoke_laguna_forward
bench_laguna_ttft
bench_laguna_pflash
bench_laguna_generate
smoke_target_forward
test_generate
test_dflash
test_model_smoke
)
foreach(_t IN LISTS _dflash_internal_h_cuda_tests)
if(TARGET ${_t})
Expand All @@ -1695,29 +1793,6 @@ if(DFLASH27B_TESTS)
endif()
endif()

if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
DFLASH27B_GPU_BACKEND STREQUAL "hip")
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_paged_attention.cpp")
add_executable(test_paged_attention test/test_paged_attention.cpp)
target_link_libraries(test_paged_attention PRIVATE
ggml ${DFLASH27B_GGML_BACKEND_TARGET} ggml-base)
if(DFLASH27B_GPU_BACKEND STREQUAL "cuda")
target_link_libraries(test_paged_attention PRIVATE CUDA::cudart)
else()
target_link_libraries(test_paged_attention PRIVATE hip::host)
endif()
target_include_directories(test_paged_attention PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src)
add_test(NAME paged_attention COMMAND test_paged_attention)
add_test(NAME paged_attention_direct
COMMAND test_paged_attention --direct)
set_tests_properties(paged_attention_direct PROPERTIES
ENVIRONMENT "GGML_CUDA_PAGED_ATTN_FORCE_PARTITIONS=1")
if(TARGET check)
add_dependencies(check test_paged_attention)
endif()
endif()
if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
DFLASH27B_GPU_BACKEND STREQUAL "hip")
AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/bench_paged_attention.cpp")
Expand Down
2 changes: 1 addition & 1 deletion server/cmake/DiscoverCppUnitTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ if(_discover_output)

file(APPEND "${CTEST_FILE}"
"add_test([==[${_ctest_name}]==] [==[${TEST_EXECUTABLE}]==] --exact [==[${_test_keyword}]==])\n"
"set_tests_properties([==[${_ctest_name}]==] PROPERTIES WORKING_DIRECTORY [==[${TEST_WORKING_DIR}]==])\n")
"set_tests_properties([==[${_ctest_name}]==] PROPERTIES WORKING_DIRECTORY [==[${TEST_WORKING_DIR}]==] SKIP_RETURN_CODE 77)\n")
list(APPEND _discovered_ctest_names "${_ctest_name}")
endforeach()
endif()
Loading
Loading