2 set(_vtk_testing_python_exe
"$<TARGET_FILE:ParaView::pvpython>")
3 set(_vtk_test_python_args --dr ${paraview_python_args})
4 vtk_add_test_python(${ARGN})
8 set(_vtk_testing_python_exe
"$<TARGET_FILE:ParaView::pvbatch>")
9 set(_vtk_test_python_args --dr ${paraview_pvbatch_args})
10 set(vtk_test_prefix
"Batch-${vtk_test_prefix}")
11 vtk_add_test_python(${ARGN})
15 set(_vtk_testing_python_exe "$<TARGET_FILE:ParaView::pvbatch>")
16 set(_vtk_test_python_args --dr ${paraview_pvbatch_args})
17 set(vtk_test_prefix
"Batch-${vtk_test_prefix}")
18 vtk_add_test_python_mpi(${ARGN})
22 if (NOT (TARGET pvserver AND TARGET pvpython))
25 set(_vtk_testing_python_exe "$<TARGET_FILE:ParaView::smTestDriver>")
26 set(_vtk_test_python_args
27 --server $<TARGET_FILE:ParaView::pvserver>
28 --client $<TARGET_FILE:ParaView::pvpython> --dr)
29 vtk_add_test_python(${ARGN})
34 "FORCE_SERIAL;FORCE_LOCK;SMTESTING_ALLOW_ERRORS" 35 "LOAD_PLUGIN;PLUGIN_PATH;CLIENT;TEST_DIRECTORY;TEST_DATA_TARGET;PREFIX;SUFFIX;_ENABLE_SUFFIX;_DISABLE_SUFFIX;BASELINE_DIR;DATA_DIRECTORY;NUMPROCS;NUMSERVERS" 36 "_COMMAND_PATTERN;LOAD_PLUGINS;PLUGIN_PATHS;TEST_SCRIPTS;TEST_NAME;ENVIRONMENT;ARGS;CLIENT_ARGS" 39 # It is kept here to avoid filtering it out from ARGN in the caller.
40 # If it were not present here, nor filtered out in the caller, then it would be considered
41 # as part of
"TEST_SCRIPTS" multivalue argument.
43 if (_paraview_add_tests_UNPARSED_ARGUMENTS)
45 "Unparsed arguments for ${function}: " 46 "${_paraview_add_tests_UNPARSED_ARGUMENTS}")
49 if (NOT DEFINED _paraview_add_tests__COMMAND_PATTERN)
51 "The `_COMMAND_PATTERN` argument is required.")
54 if (NOT DEFINED _paraview_add_tests_CLIENT)
55 if (
"__paraview_client__" IN_LIST _paraview_add_tests__COMMAND_PATTERN #
do we care?
56 AND NOT TARGET ParaView::paraview # For external testing.
57 AND NOT paraview_will_be_built) # For use within ParaView itself.
61 set(_paraview_add_tests_CLIENT
62 "$<TARGET_FILE:ParaView::paraview>")
65 if (NOT DEFINED _paraview_add_tests_PREFIX)
66 set(_paraview_add_tests_PREFIX
"pv")
69 if (NOT DEFINED _paraview_add_tests_TEST_DIRECTORY)
70 if (NOT DEFINED PARAVIEW_TEST_DIR)
71 set(_paraview_add_tests_TEST_DIRECTORY
72 "${CMAKE_BINARY_DIR}/Testing/Temporary")
74 set(_paraview_add_tests_TEST_DIRECTORY
75 "${PARAVIEW_TEST_DIR}")
76 set(_vtk_build_TEST_OUTPUT_DIRECTORY
"${PARAVIEW_TEST_DIR}")
80 if (NOT DEFINED _paraview_add_tests_DATA_DIRECTORY AND DEFINED _paraview_add_tests_default_data_directory)
81 set(_paraview_add_tests_DATA_DIRECTORY
82 "${_paraview_add_tests_default_data_directory}")
85 if (DEFINED _paraview_add_tests_TEST_SCRIPTS AND DEFINED _paraview_add_tests_TEST_NAME)
87 "Only one of `TEST_NAME` or `TEST_SCRIPTS` must be specified.")
88 elseif (NOT DEFINED _paraview_add_tests_TEST_SCRIPTS AND NOT DEFINED _paraview_add_tests_TEST_NAME)
90 "Either `TEST_SCRIPTS` or `TEST_NAME` must be specified.")
93 set(_paraview_add_tests_args
94 ${_paraview_add_tests_ARGS})
96 if (DEFINED _paraview_add_tests_PLUGIN_PATH)
97 list(APPEND _paraview_add_tests_args
98 "--plugin-search-paths=${_paraview_add_tests_PLUGIN_PATH}")
101 if (DEFINED _paraview_add_tests_PLUGIN_PATHS)
102 string(REPLACE
";" "," _plugin_paths
"${_paraview_add_tests_PLUGIN_PATHS}")
103 list(APPEND _paraview_add_tests_args
104 "--plugin-search-paths=${_plugin_paths}")
107 if (DEFINED _paraview_add_tests_LOAD_PLUGIN)
108 list(APPEND _paraview_add_tests_args
109 "--plugins=${_paraview_add_tests_LOAD_PLUGIN}")
112 if (DEFINED _paraview_add_tests_LOAD_PLUGINS)
113 string(REPLACE
";" "," _load_plugins
"${_paraview_add_tests_LOAD_PLUGINS}")
114 list(APPEND _paraview_add_tests_args
115 "--plugins=${_load_plugins}")
118 string(REPLACE
"__paraview_args__" "${_paraview_add_tests_args}" 119 _paraview_add_tests__COMMAND_PATTERN
120 "${_paraview_add_tests__COMMAND_PATTERN}")
121 string(REPLACE
"__paraview_client__" "${_paraview_add_tests_CLIENT}" 122 _paraview_add_tests__COMMAND_PATTERN
123 "${_paraview_add_tests__COMMAND_PATTERN}")
125 foreach (_paraview_add_tests_script IN LISTS _paraview_add_tests_TEST_SCRIPTS _paraview_add_tests_TEST_NAME)
126 if (DEFINED _paraview_add_tests_TEST_NAME)
127 set(_paraview_add_tests_name
"${_paraview_add_tests_script}")
128 set(_paraview_add_tests_name_base
"${_paraview_add_tests_name}")
129 set(_paraview_add_tests_script)
131 if (NOT IS_ABSOLUTE
"${_paraview_add_tests_script}")
132 set(_paraview_add_tests_script
133 "${CMAKE_CURRENT_SOURCE_DIR}/${_paraview_add_tests_script}")
135 get_filename_component(_paraview_add_tests_name
"${_paraview_add_tests_script}" NAME_WE)
136 set(_paraview_add_tests_name_base
"${_paraview_add_tests_name}")
139 string(APPEND _paraview_add_tests_name
"${_paraview_add_tests_SUFFIX}")
141 if (DEFINED _paraview_add_tests__DISABLE_SUFFIX AND ${_paraview_add_tests_name}${_paraview_add_tests__DISABLE_SUFFIX})
145 if (DEFINED _paraview_add_tests__ENABLE_SUFFIX AND NOT ${_paraview_add_tests_name}${_paraview_add_tests__ENABLE_SUFFIX})
149 if (NOT DEFINED _paraview_add_tests_TEST_DATA_TARGET)
150 if (DEFINED _paraview_add_tests_default_test_data_target)
151 set(_paraview_add_tests_TEST_DATA_TARGET
152 "${_paraview_add_tests_default_test_data_target}")
154 if (NOT DEFINED
"${_paraview_add_tests_name}_USES_DIRECT_DATA")
155 message(FATAL_ERROR
"The `TEST_DATA_TARGET` argument is required.")
160 # Build arguments to pass to smTestDriver 161 set(_paraview_add_tests_smtesting_args
"")
162 if (_paraview_add_tests_SMTESTING_ALLOW_ERRORS)
163 list(APPEND _paraview_add_tests_smtesting_args
"--allow-errors")
166 # Build arguments to pass to the clients. 167 set(_paraview_add_tests_client_args
168 "--test-directory=${_paraview_add_tests_TEST_DIRECTORY}" 169 ${_paraview_add_tests_CLIENT_ARGS})
170 if (DEFINED _paraview_add_tests_BASELINE_DIR)
171 if (DEFINED
"${_paraview_add_tests_name}_BASELINE")
172 if (DEFINED
"${_paraview_add_tests_name}_USES_DIRECT_DATA")
173 list(APPEND _paraview_add_tests_client_args
174 "--test-baseline=${_paraview_add_tests_BASELINE_DIR}/${${_paraview_add_tests_name_base}_BASELINE}")
176 list(APPEND _paraview_add_tests_client_args
177 "--test-baseline=DATA{${_paraview_add_tests_BASELINE_DIR}/${${_paraview_add_tests_name_base}_BASELINE}}")
180 if (DEFINED
"${_paraview_add_tests_name}_USES_DIRECT_DATA")
181 list(APPEND _paraview_add_tests_client_args
182 "--test-baseline=${_paraview_add_tests_BASELINE_DIR}/${_paraview_add_tests_name_base}.png")
184 list(APPEND _paraview_add_tests_client_args
185 "--test-baseline=DATA{${_paraview_add_tests_BASELINE_DIR}/${_paraview_add_tests_name_base}.png}")
188 if (NOT DEFINED
"${_paraview_add_tests_name}_USES_DIRECT_DATA")
189 ExternalData_Expand_Arguments(
"${_paraview_add_tests_TEST_DATA_TARGET}" _
190 "DATA{${_paraview_add_tests_BASELINE_DIR}/,REGEX:${_paraview_add_tests_name_base}(-.*)?(_[0-9]+)?.png}")
193 if (DEFINED
"${_paraview_add_tests_name}_THRESHOLD")
194 list(APPEND _paraview_add_tests_client_args
195 "--test-threshold=${${_paraview_add_tests_name}_THRESHOLD}")
197 if (DEFINED _paraview_add_tests_DATA_DIRECTORY)
198 list(APPEND _paraview_add_tests_client_args
199 "--data-directory=${_paraview_add_tests_DATA_DIRECTORY}")
202 if (DEFINED _paraview_add_tests_TEST_NAME)
203 string(REPLACE
"__paraview_script__" "" 204 _paraview_add_tests_script_args
205 "${_paraview_add_tests__COMMAND_PATTERN}")
206 string(REPLACE
"__paraview_scriptpath__" "" 207 _paraview_add_tests_script_args
208 "${_paraview_add_tests_script_args}")
210 string(REPLACE
"__paraview_script__" "--test-script=${_paraview_add_tests_script}" 211 _paraview_add_tests_script_args
212 "${_paraview_add_tests__COMMAND_PATTERN}")
213 string(REPLACE
"__paraview_scriptpath__" "${_paraview_add_tests_script}" 214 _paraview_add_tests_script_args
215 "${_paraview_add_tests_script_args}")
218 string(REPLACE
"__paraview_client_args__" "${_paraview_add_tests_client_args}" 219 _paraview_add_tests_script_args
220 "${_paraview_add_tests_script_args}")
221 string(REPLACE
"__paraview_test_name__" "${_paraview_add_tests_name_base}" 222 _paraview_add_tests_script_args
223 "${_paraview_add_tests_script_args}")
224 string(REPLACE
"__paraview_smtesting_args__" "${_paraview_add_tests_smtesting_args}" 225 _paraview_add_tests_script_args
226 "${_paraview_add_tests_script_args}")
229 NAME "${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 230 COMMAND ParaView::smTestDriver
232 ${_paraview_add_tests_script_args})
233 if (DEFINED
"${_paraview_add_tests_name}_USES_DIRECT_DATA")
234 add_test(${testArgs})
236 ExternalData_add_test(
"${_paraview_add_tests_TEST_DATA_TARGET}" ${testArgs})
239 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 243 # set default image compare method for all tests 244 if (DEFINED DEFAULT_USE_SSIM_IMAGE_COMP AND DEFAULT_USE_SSIM_IMAGE_COMP)
245 set(_paraview_default_image_compare_method
"VTK_TESTING_IMAGE_COMPARE_METHOD=TIGHT_VALID")
247 set(_paraview_default_image_compare_method
"VTK_TESTING_IMAGE_COMPARE_METHOD=LEGACY_VALID")
249 # METHOD can have one of the following values: 250 # ``TIGHT_VALID``: Uses euclidian type metrics to compare baselines. Baseline 251 # comparison is sensitive to outliers in this setting. 252 #``LOOSE_VALID``: Uses L1 type metrics to compare baselines. Baseline comparison 253 # is somewhat more forgiving. Typical use cases involve rendering that is highly GPU 254 # dependent, and baselines with text. 255 # ``LEGACY_VALID``: Uses legacy image compare. This metric generates a lot of 256 # false negatives. It is recommended not to use it. 258 # If this variable is not set, a test will use the the default image compare method as defined above 259 set(_paraview_image_compare_method ${_paraview_default_image_compare_method})
260 if (DEFINED
"${_paraview_add_tests_name}_METHOD")
261 if (
"${${_paraview_add_tests_name}_METHOD}" STREQUAL
"LEGACY_VALID")
262 set(_paraview_image_compare_method
"VTK_TESTING_IMAGE_COMPARE_METHOD=LEGACY_VALID")
263 elseif (
"${${_paraview_add_tests_name}_METHOD}" STREQUAL
"LOOSE_VALID")
264 set(_paraview_image_compare_method
"VTK_TESTING_IMAGE_COMPARE_METHOD=LOOSE_VALID")
265 elseif (
"${${_paraview_add_tests_name}_METHOD}" STREQUAL
"TIGHT_VALID")
266 set(_paraview_image_compare_method
"VTK_TESTING_IMAGE_COMPARE_METHOD=TIGHT_VALID")
269 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 271 ENVIRONMENT
"${_paraview_add_tests_ENVIRONMENT};${_paraview_image_compare_method}")
272 if (DEFINED
"${_paraview_add_tests_name}_TIMEOUT")
273 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 275 TIMEOUT ${${_paraview_add_tests_name}_TIMEOUT}
278 if (DEFINED _paraview_add_tests_NUMPROCS)
279 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 281 PROCESSORS
"${_paraview_add_tests_NUMPROCS}")
283 if (${_paraview_add_tests_name}_FORCE_SERIAL OR _paraview_add_tests_FORCE_SERIAL)
284 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 287 elseif (NOT DEFINED _paraview_add_tests_TEST_NAME AND EXISTS
"${_paraview_add_tests_script}")
288 # if the XML test contains PARAVIEW_TEST_ROOT we assume that we may be writing 289 # to that file and reading it back in so we add a resource lock on the XML 290 # file so that the pv.X, pvcx.X and pvcrs.X tests don't run simultaneously. 291 # we only need to do this if the test isn't forced to be serial already. 292 if (NOT IS_DIRECTORY
"${_paraview_add_tests_script}" AND NOT ${_paraview_add_tests_name}_FORCE_LOCK)
293 file(STRINGS
"${_paraview_add_tests_script}" _paraview_add_tests_paraview_test_root REGEX PARAVIEW_TEST_ROOT)
295 if (${_paraview_add_tests_name}_FORCE_LOCK OR _paraview_add_tests_paraview_test_root)
296 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 298 RESOURCE_LOCK
"${_paraview_add_tests_script}")
302 # XXX(gitlab-ci): Prevent tests from timing out after a long time. As to 303 # why the tests hang in this situation is not yet known. May be related to 304 # https://gitlab.kitware.com/paraview/paraview/-/issues/20697 305 set_property(TEST
"${_paraview_add_tests_PREFIX}.${_paraview_add_tests_name}" 307 TIMEOUT_AFTER_MATCH
"5" "Couldn't find object")
317 if (_get_prefix_PREFIX)
318 set(${varname}
"${_get_prefix_PREFIX}" PARENT_SCOPE)
320 set(${varname}
"${default}" PARENT_SCOPE)
324 # If NUMSERVERS is specified in ${ARGN} then returns in varname the number of servers to use for the test, 325 # otherwise returns the "NOT_FOUND" value. 332 if (_get_num_servers_NUMSERVERS)
333 set(${varname}
"${_get_num_servers_NUMSERVERS}" PARENT_SCOPE)
335 set(${varname}
"NOT_FOUND" PARENT_SCOPE)
342 PREFIX
"${chosen_prefix}" 343 _DISABLE_SUFFIX
"_DISABLE_C" 345 __paraview_smtesting_args__
346 --client __paraview_client__
350 __paraview_client_args__
358 set(_paraview_add_tests_args
359 PREFIX
"${chosen_prefix}" 360 _DISABLE_SUFFIX
"_DISABLE_CS" 362 __paraview_smtesting_args__
363 --server
"$<TARGET_FILE:ParaView::pvserver>" 366 --client __paraview_client__
370 __paraview_client_args__
377 if (NOT num_servers STREQUAL
"NOT_FOUND")
378 set(_paraview_add_tests_args ${_paraview_add_tests_args};
379 # Set the number of pvservers through the SMTESTDRIVER_MPI_NUMPROCS environment variable. 381 SMTESTDRIVER_MPI_NUMPROCS=${num_servers}
382 # Requires CTest to reserve NUMPROCS among the available processors 383 NUMPROCS
"${num_servers}" 392 set(_paraview_add_tests_args
393 PREFIX
"${chosen_prefix}" 394 _DISABLE_SUFFIX
"_DISABLE_CRS" 396 __paraview_smtesting_args__
397 --
data-server
"$<TARGET_FILE:ParaView::pvdataserver>" 400 --render-server
"$<TARGET_FILE:ParaView::pvrenderserver>" 403 --client __paraview_client__
407 __paraview_client_args__
413 if (NOT num_servers STREQUAL
"NOT_FOUND")
414 set(_paraview_add_tests_args ${_paraview_add_tests_args};
415 # Set the number of pvservers through the SMTESTDRIVER_MPI_NUMPROCS environment variable. 417 SMTESTDRIVER_MPI_NUMPROCS=${num_servers}
418 # Requires CTest to reserve NUMPROCS among the available processors 419 NUMPROCS
"${num_servers}" 427 _get_prefix(chosen_prefix "pvcs-multi-servers" ${ARGN})
429 PREFIX
"${chosen_prefix}" 432 __paraview_smtesting_args__
433 --test-multi-servers
"${count}" 434 --server
"$<TARGET_FILE:ParaView::pvserver>" 436 --client __paraview_client__
440 __paraview_client_args__
447 math(EXPR _paraview_add_tile_display_cpu_count
"${width} * ${height}")
449 if (_paraview_add_tile_display_cpu_count GREATER 1 AND NOT PARAVIEW_USE_MPI)
450 # we can run 1x1 tile display tests on non MPI builds. 454 _get_prefix(chosen_prefix "pvcs-tile-display" ${ARGN})
456 PREFIX
"${chosen_prefix}" 457 SUFFIX
"-${width}x${height}" 459 PV_SHARED_WINDOW_SIZE=800x600
460 SMTESTDRIVER_MPI_NUMPROCS=${_paraview_add_tile_display_cpu_count}
461 NUMPROCS
"${_paraview_add_tile_display_cpu_count}" 463 __paraview_smtesting_args__
464 --server
"$<TARGET_FILE:ParaView::pvserver>" 468 # using offscreen to avoid clobbering display (although should not be 469 # necessary) when running tests in parallel. 470 --force-offscreen-rendering
471 --client __paraview_client__
475 __paraview_client_args__
482 if (num_ranks GREATER 1 AND NOT PARAVIEW_USE_MPI)
486 get_filename_component(_config_name "${config}
" NAME_WE) 488 _get_prefix(chosen_prefix "pvcs-cave-${_config_name}
" ${ARGN}) 490 PREFIX "${chosen_prefix}
" 491 SUFFIX "-${num_ranks}
" 493 PV_SHARED_WINDOW_SIZE=400x300 494 SMTESTDRIVER_MPI_NUMPROCS=${num_ranks} 495 NUMPROCS "${num_ranks}
" 497 __paraview_smtesting_args__ 498 --server "$<TARGET_FILE:ParaView::pvserver>
" 500 # using offscreen to avoid clobbering display (although should not be 501 # necessary) when running tests in parallel. 502 --force-offscreen-rendering 505 --client __paraview_client__ 509 __paraview_client_args__ 515 # This is a catch-all function to run any custom command as the "client
" 516 # using th smTestDriver. Note, the command must print "Process started
" for 517 # smTestDriver to treat it as started otherwise the test will fail. 518 # The command to execute is passed as {ARGN} and is suffixed by each of the 519 # TEST_SCRIPTS provided, one at at time. 520 function (paraview_add_test) 521 _get_prefix(chosen_prefix "paraview
" ${ARGN}) 523 PREFIX "${chosen_prefix}
" 525 __paraview_smtesting_args__ 528 __paraview_scriptpath__ 532 # Same as `paraview_add_test` except makes smTestDriver run the command using 533 # mpi. If `PARAVIEW_USE_MPI` if not defined, this does not add any test. 534 function (paraview_add_test_mpi) 535 if (PARAVIEW_USE_MPI) 536 _get_prefix(chosen_prefix "paraview-mpi
" ${ARGN}) 538 PREFIX "${chosen_prefix}
" 539 NUMPROCS 2 # See Utilities/TestDriver/CMakeLists.txt (PARAVIEW_MPI_MAX_NUMPROCS) 541 __paraview_smtesting_args__ 544 __paraview_scriptpath__ 550 # if PARAVIEW_USE_MPI, calls paraview_add_test_mpi(), else calls 551 # paraview_add_test() 552 function (paraview_add_test_mpi_optional) 553 if (PARAVIEW_USE_MPI) 554 paraview_add_test_mpi(${ARGN}) 556 paraview_add_test(${ARGN})
function paraview_add_client_server_render_tests()
function paraview_add_test_pvbatch()
function paraview_add_test_python()
function paraview_add_client_tests()
function paraview_add_test_pvbatch_mpi()
function _get_prefix(varname, default)
function paraview_add_test()
function paraview_add_test_driven()
function _paraview_add_tests(function)
function _get_num_servers(varname)
function paraview_add_multi_server_tests(count)
function paraview_add_cave_tests(num_ranks, config)
function paraview_add_tile_display_tests(width, height)
function paraview_add_client_server_tests()
function paraview_add_test_mpi()