Add stronger infrastructure for CMake-only tests

The CMakeOnly directory added by commit 9a20abf0 (Add infrastructure for
CMake-only tests, 2012-01-11) was sufficient only for tests that always
run CMake to successfully configure a project.  Later commit eeaaffcb
(find_package: Test error and warning messages in failure cases,
2012-02-28) added a sample test that covers failure cases.

Generalize the above to create new "RunCMake" test infrastructure that
can run CMake multiple times for a single project with different
variations and check for expected result/stdout/stderr.  Allow for both
successful and failing CMake project configuration cases.  This will be
useful to test error messages and failure behavior.
This commit is contained in:
Brad King 2012-03-09 16:24:43 -05:00
parent c7bdef5b48
commit 42a81e7119
4 changed files with 120 additions and 4 deletions

View File

@ -54,6 +54,7 @@ IF(BUILD_TESTING)
ADD_SUBDIRECTORY(CMakeLib) ADD_SUBDIRECTORY(CMakeLib)
ADD_SUBDIRECTORY(CMakeOnly) ADD_SUBDIRECTORY(CMakeOnly)
ADD_SUBDIRECTORY(CMakeCommands) ADD_SUBDIRECTORY(CMakeCommands)
ADD_SUBDIRECTORY(RunCMake)
ADD_SUBDIRECTORY(FindPackageModeMakefileTest) ADD_SUBDIRECTORY(FindPackageModeMakefileTest)

View File

@ -16,10 +16,15 @@ your test to the test runs.
This includes tests that will build something using try_compile() and friends, This includes tests that will build something using try_compile() and friends,
but nothing that expects add_executable(), add_library(), or add_test() to run. but nothing that expects add_executable(), add_library(), or add_test() to run.
If this matches your test you should put it into the Tests/CMakeOnly/ directory. If the test configures the project only once and it must succeed then put it
Create a subdirectory named like your test and write the CMakeLists.txt you into the Tests/CMakeOnly/ directory. Create a subdirectory named like your
need into that subdirectory. Use the add_CMakeOnly_test() macro from test and write the CMakeLists.txt you need into that subdirectory. Use the
Tests/CMakeOnly/CMakeLists.txt to add your test to the test runs. add_CMakeOnly_test() macro from Tests/CMakeOnly/CMakeLists.txt to add your
test to the test runs.
If the test configures the project with multiple variations and verifies
success or failure each time then put it into the Tests/RunCMake/ directory.
Read the instructions in Tests/RunCMake/CMakeLists.txt to add a test.
3. If you are testing something from the Modules directory 3. If you are testing something from the Modules directory

View File

@ -0,0 +1,41 @@
# This directory contains tests that run CMake to configure a project
# but do not actually build anything. To add a test:
#
# 1.) Add a subdirectory named for the test.
#
# 2.) Call add_RunCMake_test and pass the test directory name.
#
# 3.) Create a RunCMakeTest.cmake script in the directory containing
# include(RunCMake)
# run_cmake(SubTest1)
# ...
# run_cmake(SubTestN)
# where SubTest1..SubTestN are sub-test names each corresponding to
# an independent CMake run and project configuration.
#
# 3.) Create a CMakeLists.txt file in the directory containing
# cmake_minimum_required(...)
# project(${RunCMake_TEST} NONE) # or languages needed
# include(${RunCMake_TEST}.cmake)
# where "${RunCMake_TEST}" is literal. A value for RunCMake_TEST
# will be passed to CMake by the run_cmake macro when running each
# sub-test.
#
# 4.) Create a <SubTest>.cmake file for each sub-test named above
# containing the actual test code. Optionally create files
# containing expected test results:
# <SubTest>-result.txt = Process result expected if not "0"
# <SubTest>-stdout.txt = Regex matching expected stdout content
# <SubTest>-stderr.txt = Regex matching expected stderr content
# Note that trailing newlines will be stripped from actual test
# output before matching against the stdout and stderr expressions.
macro(add_RunCMake_test test)
add_test(RunCMake.${test} ${CMAKE_CMAKE_COMMAND}
-DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
-DRunCMake_GENERATOR=${CMAKE_TEST_GENERATOR}
-DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
-DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${test}
-P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
)
endmacro()

View File

@ -0,0 +1,69 @@
foreach(arg
RunCMake_GENERATOR
RunCMake_SOURCE_DIR
RunCMake_BINARY_DIR
)
if(NOT DEFINED ${arg})
message(FATAL_ERROR "${arg} not given!")
endif()
endforeach()
function(run_cmake test)
set(top_src "${RunCMake_SOURCE_DIR}")
set(top_bin "${RunCMake_BINARY_DIR}")
if(EXISTS ${top_src}/${test}-result.txt)
file(READ ${top_src}/${test}-result.txt expect_result)
string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
else()
set(expect_result 0)
endif()
foreach(o out err)
if(EXISTS ${top_src}/${test}-std${o}.txt)
file(READ ${top_src}/${test}-std${o}.txt expect_std${o})
string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
else()
unset(expect_std${o})
endif()
endforeach()
set(source_dir "${top_src}")
set(binary_dir "${top_bin}/${test}-build")
file(REMOVE_RECURSE "${binary_dir}")
file(MAKE_DIRECTORY "${binary_dir}")
execute_process(
COMMAND ${CMAKE_COMMAND} "${source_dir}"
-G "${RunCMake_GENERATOR}" -DRunCMake_TEST=${test}
WORKING_DIRECTORY "${binary_dir}"
OUTPUT_VARIABLE actual_stdout
ERROR_VARIABLE actual_stderr
RESULT_VARIABLE actual_result
)
set(msg "")
if(NOT "${actual_result}" STREQUAL "${expect_result}")
set(msg "${msg}Result is [${actual_result}], not [${expect_result}].\n")
endif()
foreach(o out err)
string(REGEX REPLACE "\n+$" "" actual_std${o} "${actual_std${o}}")
set(expect_${o} "")
if(DEFINED expect_std${o})
if(NOT "${actual_std${o}}" MATCHES "${expect_std${o}}")
string(REGEX REPLACE "\n" "\n expect-${o}> " expect_${o}
" expect-${o}> ${expect_std${o}}")
set(expect_${o} "Expected std${o} to match:\n${expect_${o}}\n")
set(msg "${msg}std${o} does not match that expected.\n")
endif()
endif()
endforeach()
if(msg)
string(REGEX REPLACE "\n" "\n actual-out> " actual_out " actual-out> ${actual_stdout}")
string(REGEX REPLACE "\n" "\n actual-err> " actual_err " actual-err> ${actual_stderr}")
message(SEND_ERROR "${test} - FAILED:\n"
"${msg}"
"${expect_out}"
"Actual stdout:\n${actual_out}\n"
"${expect_err}"
"Actual stderr:\n${actual_err}\n"
)
else()
message(STATUS "${test} - PASSED")
endif()
endfunction()