4 VTK uses the [ExternalData][] CMake module to handle the
data management
for 5 its test suite.
Test data is only downloaded when a test which requires it is
6 enabled and it is cached so that every build does not need to redownload the
9 To facilitate
this workflow, there are a number of CMake functions available in
10 order to indicate that test
data is required.
16 get_filename_component(_vtkModuleTesting_dir
"${CMAKE_CURRENT_LIST_FILE}" DIRECTORY)
21 Data may be downloaded manually
using this function:
27 This will download
data inside of the input
data directory
for the modules
28 being built at that
time (see the `TEST_INPUT_DATA_DIRECTORY` argument of
31 For supported `PATHSPEC` syntax, see the
32 [associated
documentation][ExternalData pathspecs] in `ExternalData`. These
33 arguments are already wrapped in the `DATA{}` syntax and are assumed to be
34 relative paths from the input
data directory.
36 [ExternalData pathspecs]: TODO
40 foreach (arg IN LISTS ARGN)
41 if (IS_ABSOLUTE
"${arg}")
46 "DATA{${_vtk_build_TEST_INPUT_DATA_DIRECTORY}/${arg}}")
50 ExternalData_Expand_Arguments("${_vtk_build_TEST_DATA_TARGET}
" _ ${data_args}) 54 ## Creating test executables 56 This function creates an executable from the list of sources passed to it. It 57 is automatically linked to the module the tests are intended for as well as any 58 declared test dependencies of the module. 61 vtk_module_test_executable(<NAME> <SOURCE>...) 64 This function is not usually used directly, but instead through the other 65 convenience functions. 67 function (vtk_module_test_executable name) 68 add_executable("${
name}
" ${ARGN}) 69 get_property(test_depends GLOBAL 70 PROPERTY "_vtk_module_${_vtk_build_test}_test_depends
") 71 get_property(test_optional_depends GLOBAL 72 PROPERTY "_vtk_module_${_vtk_build_test}_test_optional_depends
") 73 set(optional_depends_flags) 74 foreach (test_optional_depend IN LISTS test_optional_depends) 75 if (TARGET "${test_optional_depend}
") 76 list(APPEND test_depends 77 "${test_optional_depend}
") 78 set(test_optional_depend_flag "1
") 80 set(test_optional_depend_flag "0
") 82 string(REPLACE "::
" "_
" safe_test_optional_depend "${test_optional_depend}
") 83 list(APPEND optional_depends_flags 84 "VTK_MODULE_ENABLE_${safe_test_optional_depend}=${test_optional_depend_flag}
") 87 target_link_libraries("${
name}
" 91 target_compile_definitions("${
name}
" 93 ${optional_depends_flags}) 97 MODULES "${_vtk_build_test}
" 104 Test names default to using the basename of the filename which contains the 105 test. Two tests may share the same file by prefixing with a custom name for the 108 The two parsed syntaxes are: 110 - `CustomTestName,TestFile` 113 Note that `TestFile` should already have had its extension stripped (usually 114 done by `_vtk_test_parse_args`). 116 In general, the name of a test will be `<EXENAME>-<TESTNAME>`, however, by 117 setting `vtk_test_prefix`, the test name will instead be 118 `<EXENAME>-<PREFIX><TESTNAME>`. 122 This function parses the name from a testspec. The calling scope has 123 `test_name` and `test_file` variables set in it. 126 _vtk_test_parse_name(<TESTSPEC>) 129 function (_vtk_test_parse_name name) 130 if (name AND name MATCHES "^([^,]*),(.*)$
") 131 set(test_name "${CMAKE_MATCH_1}
" PARENT_SCOPE) 132 set(test_file "${CMAKE_MATCH_2}
" PARENT_SCOPE) 134 set(test_name "${
name}
" PARENT_SCOPE) 135 set(test_file "${
name}
" PARENT_SCOPE) 140 ## Test function arguments 142 Each test is specified using one of the two following syntaxes 144 - `<NAME>.<SOURCE_EXT>` 145 - `<NAME>.<SOURCE_EXT>,<OPTIONS>` 147 Where `NAME` is a valid test name. If present, the specified `OPTIONS` are only 148 for the associated test. The expected extension is specified by the associated 153 Given a list of valid "options
", this function will parse out a the following 156 - `args`: Unrecognized arguments. These should be interpreted as arguments 157 that should be passed on the command line to all tests in this parse group. 158 - `options`: Options specified globally (for all tests in this group). 159 - `names`: A list containing all named tests. These should be parsed by 160 `_vtk_test_parse_name`. 161 - `_<NAME>_options`: Options specific to a certain test. 164 _vtk_test_parse_args(<OPTIONS> <SOURCE_EXT> <ARG>...) 167 In order to be recognized as a source file, the `SOURCE_EXT` must be used. 168 Without it, all non-option arguments are placed into `args`. Each test is 169 parsed out matching these: 171 function (_vtk_test_parse_args options source_ext) 176 foreach (arg IN LISTS ARGN) 178 foreach (option IN LISTS options) 179 if (arg STREQUAL option) 180 list(APPEND global_options "${option}
") 187 elseif (source_ext AND arg MATCHES "^([^.]*)\\.${source_ext},?(.*)$
") 188 set(name "${CMAKE_MATCH_1}
") 189 string(REPLACE ",
" ";
" "_${
name}_options
" "${CMAKE_MATCH_2}
") 190 list(APPEND names "${
name}
") 192 list(APPEND args "${arg}
") 196 foreach (name IN LISTS names) 197 set("_${
name}_options
" "${_${
name}_options}
" 200 set(options "${global_options}
" 209 For handling global option settings, this function sets variables in the 210 calling scoped named `<PREFIX><OPTION>` to either `0` or `1` if the option is 211 present in the remaining argument list. 214 _vtk_test_set_options(<OPTIONS> <PREFIX> <ARG>...) 217 Additionally, a non-`0` default for a given option may be specified by a 218 variable with the same name as the option and specifying a prefix for the 221 function (_vtk_test_set_options options prefix) 222 foreach (option IN LISTS options) 225 set(default "${${option}}
") 227 set("${prefix}${option}
" "${
default}
" 230 foreach (option IN LISTS ARGN) 231 set("${prefix}${option}
" 1 236 # If set, use the maximum number of processors for tests. Otherwise, just use 1 237 # processor by default. 238 set(VTK_MPI_NUMPROCS "2
" CACHE STRING 239 "Number of processors available to run parallel tests.
") 240 # Hide the variable if we don't have `MPIEXEC_EXECUTABLE` anyways. 241 if (MPIEXEC_EXECUTABLE) 242 set(_vtk_mpi_max_numprocs_type STRING) 244 set(_vtk_mpi_max_numprocs_type INTERNAL) 246 set_property(CACHE VTK_MPI_NUMPROCS 248 TYPE "${_vtk_mpi_max_numprocs_type}
") 253 This function declares C++ tests. Source files are required to use the `cxx` 257 vtk_add_test_cxx(<EXENAME> <VARNAME> <ARG>...) 260 Each argument should be either an option, a test specification, or it is passed 261 as flags to all tests declared in the group. The list of tests is set in the 262 `<VARNAME>` variable in the calling scope. 266 - `NO_DATA`: The test does not need to know the test input data directory. If 267 it does, it is passed on the command line via the `-D` flag. 268 - `NO_VALID`: The test does not have a valid baseline image. If it does, the 269 baseline is assumed to be in `../Data/Baseline/<NAME>.png` relative to the 270 current source directory. If alternate baseline images are required, 271 `<NAME>` may be suffixed by `_1`, `_2`, etc. The valid image is passed via 273 - `NO_OUTPUT`: The test does not need to write out any data to the 274 filesystem. If it does, a directory which may be written to is passed via 277 Additional flags may be passed to tests using the `${_vtk_build_test}_ARGS` 278 variable or the `<NAME>_ARGS` variable. 280 function (vtk_add_test_cxx exename _tests) 285 _vtk_test_parse_args("${cxx_options}
" "cxx
" ${ARGN}) 286 _vtk_test_set_options("${cxx_options}
" "" ${options}) 288 set(_vtk_fail_regex "(\n|^)ERROR:
" "instance(s)? still around
") 290 foreach (name IN LISTS names) 291 _vtk_test_set_options("${cxx_options}
" "local_
" ${_${name}_options}) 292 _vtk_test_parse_name("${
name}
") 295 if (NOT local_NO_DATA) 296 set(_D -D "${_vtk_build_TEST_OUTPUT_DATA_DIRECTORY}
") 300 if (NOT local_NO_OUTPUT) 301 set(_T -T "${_vtk_build_TEST_OUTPUT_DIRECTORY}
") 305 if (NOT local_NO_VALID) 306 set(_V -V "DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Data/Baseline/${test_name}.png,:}
") 310 VTK_SERIAL_TESTS_USE_MPIEXEC) 311 set(_vtk_test_cxx_pre_args 312 "${MPIEXEC_EXECUTABLE}
" 313 "${MPIEXEC_NUMPROC_FLAG}
" "1
" 317 ExternalData_add_test("${_vtk_build_TEST_DATA_TARGET}
" 318 NAME "${_vtk_build_test}Cxx-${vtk_test_prefix}${test_name}
" 319 COMMAND "${_vtk_test_cxx_pre_args}
" "$<TARGET_FILE:${exename}>
" 322 ${${_vtk_build_test}_ARGS} 325 set_tests_properties("${_vtk_build_test}Cxx-${vtk_test_prefix}${test_name}
" 327 LABELS "${_vtk_build_test_labels}
" 328 FAIL_REGULAR_EXPRESSION "${_vtk_fail_regex}
" 329 # This must match VTK_SKIP_RETURN_CODE in vtkTestingObjectFactory.h 333 list(APPEND ${_tests} "${test_file}
") 336 set("${_tests}
" ${${_tests}} PARENT_SCOPE) 342 This function declares C++ tests which should be run under an MPI environment. 343 Source files are required to use the `cxx` extension. 346 vtk_add_test_mpi(<EXENAME> <VARNAME> <ARG>...) 349 Each argument should be either an option, a test specification, or it is passed 350 as flags to all tests declared in the group. The list of tests is set in the 351 `<VARNAME>` variable in the calling scope. 356 - `NO_VALID`: The test does not have a valid baseline image. If it does, the 357 baseline is assumed to be in `../Data/Baseline/<NAME>.png` relative to the 358 current source directory. If alternate baseline images are required, 359 `<NAME>` may be suffixed by `_1`, `_2`, etc. The valid image is passed via 362 Each test is run using the number of processors specified by the following 363 variables (using the first one which is set): 366 - `<EXENAME>_NUMPROCS` 367 - `VTK_MPI_NUMPROCS` (defaults to `2`) 369 Additional flags may be passed to tests using the `${_vtk_build_test}_ARGS` 370 variable or the `<NAME>_ARGS` variable. 372 function (vtk_add_test_mpi exename _tests) 377 _vtk_test_parse_args("${mpi_options}
" "cxx
" ${ARGN}) 378 _vtk_test_set_options("${mpi_options}
" "" ${options}) 380 set(_vtk_fail_regex "(\n|^)ERROR:
" "instance(s)? still around
") 382 set(default_numprocs ${VTK_MPI_NUMPROCS}) 383 if (${exename}_NUMPROCS) 384 set(default_numprocs ${${exename}_NUMPROCS}) 387 foreach (name IN LISTS names) 388 _vtk_test_set_options("${mpi_options}
" "local_
" ${_${name}_options}) 389 _vtk_test_parse_name(${name}) 394 if (local_TESTING_DATA) 395 set(_D -D "${_vtk_build_TEST_OUTPUT_DATA_DIRECTORY}
") 396 set(_T -T "${_vtk_build_TEST_OUTPUT_DIRECTORY}
") 398 if (NOT local_NO_VALID) 399 set(_V -V "DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Data/Baseline/${
name}.png,:}
") 403 set(numprocs ${default_numprocs}) 404 if (${name}_NUMPROCS) 405 set(numprocs "${${
name}_NUMPROCS}
") 408 ExternalData_add_test("${_vtk_build_TEST_DATA_TARGET}
" 409 NAME "${_vtk_build_test}Cxx-MPI-${vtk_test_prefix}${test_name}
" 410 COMMAND "${MPIEXEC_EXECUTABLE}
" 411 "${MPIEXEC_NUMPROC_FLAG}
" "${numprocs}
" 413 "$<TARGET_FILE:${exename}>
" 417 ${${_vtk_build_test}_ARGS} 419 ${MPIEXEC_POSTFLAGS}) 420 set_tests_properties("${_vtk_build_test}Cxx-MPI-${vtk_test_prefix}${test_name}
" 422 LABELS "${_vtk_build_test_labels}
" 423 PROCESSORS "${numprocs}
" 424 FAIL_REGULAR_EXPRESSION "${_vtk_fail_regex}
" 425 # This must match VTK_SKIP_RETURN_CODE in vtkTestingObjectFactory.h" 428 set_property(TEST
"${_vtk_build_test}Cxx-MPI-${vtk_test_prefix}${test_name}" APPEND
430 REQUIRED_FILES
"$<TARGET_FILE:${exename}>")
431 list(APPEND ${_tests}
"${test_file}")
434 set(${_tests} ${${_tests}} PARENT_SCOPE)
438 ### C++ test executable 444 Creates an executable named `EXENAME` which contains the tests listed in the
445 variable named in the `VARNAME` argument. The `EXENAME` must match the
446 `EXENAME` passed to the test declarations when building the list of tests.
448 If `RENDERING_FACTORY` is provided, VTK
's rendering factories are initialized 451 Any additional arguments are added as additional sources for the executable. 453 function (vtk_test_cxx_executable exename _tests) 456 _vtk_test_parse_args("${exe_options}" "" ${ARGN}) 457 _vtk_test_set_options("${exe_options}" "" ${options}) 460 # No tests -> no need for an executable. 464 if (RENDERING_FACTORY) 465 include("${_vtkModuleTesting_dir}/vtkTestingRenderingDriver.cmake") 466 set(test_driver vtkTestingObjectFactory.h) 468 include("${_vtkModuleTesting_dir}/vtkTestingDriver.cmake") 469 set(test_driver vtkTestDriver.h) 472 set(extra_sources ${args}) 474 create_test_sourcelist(test_sources "${exename}.cxx" ${${_tests}} 475 EXTRA_INCLUDE "${test_driver}") 478 vtk_module_test_executable("${exename}" ${test_sources} ${extra_sources}) 480 message(FATAL_ERROR "_vtk_build_test is not set!") 485 MPI executables used to have their own test executable function. This is no 486 longer necessary and is deprecated. Instead, `vtk_test_cxx_executable` should 489 function (vtk_test_mpi_executable exename _tests) 491 "The `vtk_test_mpi_executable` function is deprecated; use " 492 "`vtk_test_cxx_executable` instead.") 493 vtk_test_cxx_executable("${exename}" "${_tests}" ${ARGN}) 499 This function declares Python tests. Test files are required to use the `py` 503 vtk_add_test_python(<EXENAME> <VARNAME> <ARG>...) 508 If the `_vtk_testing_python_exe` variable is not set, the `vtkpython` binary is 509 used by default. Additional arguments may be passed in this variable as well. 521 Each argument should be either an option, a test specification, or it is passed 522 as flags to all tests declared in the group. The list of tests is set in the 523 `<VARNAME>` variable in the calling scope. 527 - `NO_DATA`: The test does not need to know the test input data directory. If 528 it does, it is passed on the command line via the `-D` flag. 529 - `NO_OUTPUT`: The test does not need to write out any data to the 530 filesystem. If it does, a directory which may be written to is passed via 532 - `NO_VALID`: The test does not have a valid baseline image. If it does, the 533 baseline is assumed to be in `../Data/Baseline/<NAME>.png` relative to the 534 current source directory. If alternate baseline images are required, 535 `<NAME>` may be suffixed by `_1`, `_2`, etc. The valid image is passed via 537 - `NO_RT`: If `NO_RT` is specified, `-B` is passed instead of `-V`, only 538 providing a baseline dir, assuming `NO_VALID` is not specified. 539 - `DIRECT_DATA` : If `DIRECT_DATA` is specified, the baseline path will be provided 540 as is, without the use of ExternalData_add_test. 541 - `JUST_VALID`: Only applies when both `NO_VALID` and `NO_RT` are not 542 present. If it is not specified, `-A` is passed with path to the directory 543 of the `vtkTclTest2Py` Python package and the test is run via the 544 `rtImageTest.py` script. Note that this currently only works when building 545 against a VTK build tree; the VTK install tree does not include this script 546 or its associated Python package. 548 Additional flags may be passed to tests using the `${_vtk_build_test}_ARGS` 549 variable or the `<NAME>_ARGS` variable. 551 Note that the `vtkTclTest2Py` support will eventually be removed. It is a 552 legacy of the conversion of many tests from Tcl to Python. 554 function (vtk_add_test_python) 555 if (NOT _vtk_testing_python_exe) 556 set(_vtk_testing_python_exe "$<TARGET_FILE:VTK::vtkpython>") 566 _vtk_test_parse_args("${python_options}" "py" ${ARGN}) 567 _vtk_test_set_options("${python_options}" "" ${options}) 569 set(_vtk_fail_regex "(\n|^)ERROR: " "instance(s)? still around") 571 foreach (name IN LISTS names) 572 _vtk_test_set_options("${python_options}" "local_" ${_${name}_options}) 573 _vtk_test_parse_name(${name}) 576 if (NOT local_NO_DATA) 577 set(_D -D "${_vtk_build_TEST_OUTPUT_DATA_DIRECTORY}") 584 if (NOT local_NO_VALID) 586 if (local_DIRECT_DATA) 587 set(_B -B "${CMAKE_CURRENT_SOURCE_DIR}/Data/Baseline/") 589 set(_B -B "DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Data/Baseline/,REGEX:${test_name}(-.*)?(_[0-9]+)?.png}") 592 if (local_DIRECT_DATA) 593 set(_V -V "${CMAKE_CURRENT_SOURCE_DIR}/Data/Baseline/${test_name}.png") 595 set(_V -V "DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Data/Baseline/${test_name}.png,:}") 597 if (NOT local_JUST_VALID) 598 # TODO: This should be fixed to also work from an installed VTK. 599 set(rtImageTest "${VTK_SOURCE_DIR}/Utilities/vtkTclTest2Py/rtImageTest.py") 600 set(_A -A "${VTK_SOURCE_DIR}/Utilities/vtkTclTest2Py") 606 if (NOT local_NO_OUTPUT) 607 set(_T -T "${_vtk_build_TEST_OUTPUT_DIRECTORY}") 610 if (NOT _vtk_build_TEST_FILE_DIRECTORY) 611 set(_vtk_build_TEST_FILE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 615 VTK_SERIAL_TESTS_USE_MPIEXEC AND 616 NOT DEFINED _vtk_test_python_pre_args) 617 set(_vtk_test_python_pre_args 618 "${MPIEXEC_EXECUTABLE}" 619 "${MPIEXEC_NUMPROC_FLAG}" "1" 622 set(testArgs NAME "${_vtk_build_test}Python${_vtk_test_python_suffix}-${vtk_test_prefix}${test_name}" 623 COMMAND ${_vtk_test_python_pre_args} 624 "${_vtk_testing_python_exe}" ${_vtk_test_python_args} --enable-bt 626 "${_vtk_build_TEST_FILE_DIRECTORY}/${test_file}.py" 628 ${${_vtk_build_test}_ARGS} 630 ${_D} ${_B} ${_T} ${_V} ${_A}) 632 if (local_DIRECT_DATA) 633 add_test(${testArgs}) 635 ExternalData_add_test("${_vtk_build_TEST_DATA_TARGET}" ${testArgs}) 638 set_tests_properties("${_vtk_build_test}Python${_vtk_test_python_suffix}-${vtk_test_prefix}${test_name}" 640 LABELS "${_vtk_build_test_labels}" 641 FAIL_REGULAR_EXPRESSION "${_vtk_fail_regex}" 642 # This must match the skip() function in vtk/test/Testing.py" 647 set_tests_properties("${_vtk_build_test}Python${_vtk_test_python_suffix}-${vtk_test_prefix}${test_name}" 649 PROCESSORS "${numprocs}") 657 A small wrapper around `vtk_add_test_python` which adds support for running 658 MPI-aware tests written in Python. 660 The `$<module library name>_NUMPROCS` variable may be used to use a non-default 661 number of processors for a test. 663 This forces running with the `pvtkpython` executable. 665 function (vtk_add_test_python_mpi) 666 set(_vtk_test_python_suffix "-MPI") 668 set(numprocs "${VTK_MPI_NUMPROCS}") 669 _vtk_module_get_module_property("${_vtk_build_test}" 670 PROPERTY "library_name" 671 VARIABLE _vtk_test_python_library_name) 672 if (${_vtk_test_python_library_name}_NUMPROCS) 673 set(numprocs "${${_vtk_test_python_library_name}_NUMPROCS}") 676 set(_vtk_test_python_pre_args 677 "${MPIEXEC_EXECUTABLE}" 678 "${MPIEXEC_NUMPROC_FLAG}" "${numprocs}" 681 if (NOT _vtk_testing_python_exe) 682 set(_vtk_testing_python_exe "$<TARGET_FILE:VTK::pvtkpython>") 684 vtk_add_test_python(${ARGN})
function vtk_module_build()
Build modules and kits.
function vtk_test_cxx_executable(exename, _tests)
.md C++ test executable
function vtk_module_test_data()
.md vtkModuleTesting