Add stronger infrastructure for CMake-only tests
The CMakeOnly directory added by commit9a20abf0(Add infrastructure for CMake-only tests, 2012-01-11) was sufficient only for tests that always run CMake to successfully configure a project. Later commiteeaaffcb(find_package: Test error and warning messages in failure cases, 2012-02-28) added a sample test that covers failure cases. Generalize the above to create new "RunCMake" test infrastructure that can run CMake multiple times for a single project with different variations and check for expected result/stdout/stderr. Allow for both successful and failing CMake project configuration cases. This will be useful to test error messages and failure behavior.
This commit is contained in:
Brad King
parent
c7bdef5b48
commit
42a81e7119
|
|
@ -54,6 +54,7 @@ IF(BUILD_TESTING)
|
|||
ADD_SUBDIRECTORY(CMakeLib)
|
||||
ADD_SUBDIRECTORY(CMakeOnly)
|
||||
ADD_SUBDIRECTORY(CMakeCommands)
|
||||
ADD_SUBDIRECTORY(RunCMake)
|
||||
|
||||
ADD_SUBDIRECTORY(FindPackageModeMakefileTest)
|
||||
|
||||
|
|
|
|||
13
Tests/README
13
Tests/README
|
|
@ -16,10 +16,15 @@ your test to the test runs.
|
|||
This includes tests that will build something using try_compile() and friends,
|
||||
but nothing that expects add_executable(), add_library(), or add_test() to run.
|
||||
|
||||
If this matches your test you should put it into the Tests/CMakeOnly/ directory.
|
||||
Create a subdirectory named like your test and write the CMakeLists.txt you
|
||||
need into that subdirectory. Use the add_CMakeOnly_test() macro from
|
||||
Tests/CMakeOnly/CMakeLists.txt to add your test to the test runs.
|
||||
If the test configures the project only once and it must succeed then put it
|
||||
into the Tests/CMakeOnly/ directory. Create a subdirectory named like your
|
||||
test and write the CMakeLists.txt you need into that subdirectory. Use the
|
||||
add_CMakeOnly_test() macro from Tests/CMakeOnly/CMakeLists.txt to add your
|
||||
test to the test runs.
|
||||
|
||||
If the test configures the project with multiple variations and verifies
|
||||
success or failure each time then put it into the Tests/RunCMake/ directory.
|
||||
Read the instructions in Tests/RunCMake/CMakeLists.txt to add a test.
|
||||
|
||||
3. If you are testing something from the Modules directory
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
# This directory contains tests that run CMake to configure a project
|
||||
# but do not actually build anything. To add a test:
|
||||
#
|
||||
# 1.) Add a subdirectory named for the test.
|
||||
#
|
||||
# 2.) Call add_RunCMake_test and pass the test directory name.
|
||||
#
|
||||
# 3.) Create a RunCMakeTest.cmake script in the directory containing
|
||||
# include(RunCMake)
|
||||
# run_cmake(SubTest1)
|
||||
# ...
|
||||
# run_cmake(SubTestN)
|
||||
# where SubTest1..SubTestN are sub-test names each corresponding to
|
||||
# an independent CMake run and project configuration.
|
||||
#
|
||||
# 3.) Create a CMakeLists.txt file in the directory containing
|
||||
# cmake_minimum_required(...)
|
||||
# project(${RunCMake_TEST} NONE) # or languages needed
|
||||
# include(${RunCMake_TEST}.cmake)
|
||||
# where "${RunCMake_TEST}" is literal. A value for RunCMake_TEST
|
||||
# will be passed to CMake by the run_cmake macro when running each
|
||||
# sub-test.
|
||||
#
|
||||
# 4.) Create a <SubTest>.cmake file for each sub-test named above
|
||||
# containing the actual test code. Optionally create files
|
||||
# containing expected test results:
|
||||
# <SubTest>-result.txt = Process result expected if not "0"
|
||||
# <SubTest>-stdout.txt = Regex matching expected stdout content
|
||||
# <SubTest>-stderr.txt = Regex matching expected stderr content
|
||||
# Note that trailing newlines will be stripped from actual test
|
||||
# output before matching against the stdout and stderr expressions.
|
||||
|
||||
macro(add_RunCMake_test test)
|
||||
add_test(RunCMake.${test} ${CMAKE_CMAKE_COMMAND}
|
||||
-DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
|
||||
-DRunCMake_GENERATOR=${CMAKE_TEST_GENERATOR}
|
||||
-DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
|
||||
-DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${test}
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
|
||||
)
|
||||
endmacro()
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
foreach(arg
|
||||
RunCMake_GENERATOR
|
||||
RunCMake_SOURCE_DIR
|
||||
RunCMake_BINARY_DIR
|
||||
)
|
||||
if(NOT DEFINED ${arg})
|
||||
message(FATAL_ERROR "${arg} not given!")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
function(run_cmake test)
|
||||
set(top_src "${RunCMake_SOURCE_DIR}")
|
||||
set(top_bin "${RunCMake_BINARY_DIR}")
|
||||
if(EXISTS ${top_src}/${test}-result.txt)
|
||||
file(READ ${top_src}/${test}-result.txt expect_result)
|
||||
string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
|
||||
else()
|
||||
set(expect_result 0)
|
||||
endif()
|
||||
foreach(o out err)
|
||||
if(EXISTS ${top_src}/${test}-std${o}.txt)
|
||||
file(READ ${top_src}/${test}-std${o}.txt expect_std${o})
|
||||
string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
|
||||
else()
|
||||
unset(expect_std${o})
|
||||
endif()
|
||||
endforeach()
|
||||
set(source_dir "${top_src}")
|
||||
set(binary_dir "${top_bin}/${test}-build")
|
||||
file(REMOVE_RECURSE "${binary_dir}")
|
||||
file(MAKE_DIRECTORY "${binary_dir}")
|
||||
execute_process(
|
||||
COMMAND ${CMAKE_COMMAND} "${source_dir}"
|
||||
-G "${RunCMake_GENERATOR}" -DRunCMake_TEST=${test}
|
||||
WORKING_DIRECTORY "${binary_dir}"
|
||||
OUTPUT_VARIABLE actual_stdout
|
||||
ERROR_VARIABLE actual_stderr
|
||||
RESULT_VARIABLE actual_result
|
||||
)
|
||||
set(msg "")
|
||||
if(NOT "${actual_result}" STREQUAL "${expect_result}")
|
||||
set(msg "${msg}Result is [${actual_result}], not [${expect_result}].\n")
|
||||
endif()
|
||||
foreach(o out err)
|
||||
string(REGEX REPLACE "\n+$" "" actual_std${o} "${actual_std${o}}")
|
||||
set(expect_${o} "")
|
||||
if(DEFINED expect_std${o})
|
||||
if(NOT "${actual_std${o}}" MATCHES "${expect_std${o}}")
|
||||
string(REGEX REPLACE "\n" "\n expect-${o}> " expect_${o}
|
||||
" expect-${o}> ${expect_std${o}}")
|
||||
set(expect_${o} "Expected std${o} to match:\n${expect_${o}}\n")
|
||||
set(msg "${msg}std${o} does not match that expected.\n")
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
if(msg)
|
||||
string(REGEX REPLACE "\n" "\n actual-out> " actual_out " actual-out> ${actual_stdout}")
|
||||
string(REGEX REPLACE "\n" "\n actual-err> " actual_err " actual-err> ${actual_stderr}")
|
||||
message(SEND_ERROR "${test} - FAILED:\n"
|
||||
"${msg}"
|
||||
"${expect_out}"
|
||||
"Actual stdout:\n${actual_out}\n"
|
||||
"${expect_err}"
|
||||
"Actual stderr:\n${actual_err}\n"
|
||||
)
|
||||
else()
|
||||
message(STATUS "${test} - PASSED")
|
||||
endif()
|
||||
endfunction()
|
||||
Loading…
Reference in New Issue