Add stronger infrastructure for CMake-only tests

The CMakeOnly directory added by commit 9a20abf0 (Add infrastructure for
CMake-only tests, 2012-01-11) was sufficient only for tests that always
run CMake to successfully configure a project.  Later commit eeaaffcb
(find_package: Test error and warning messages in failure cases,
2012-02-28) added a sample test that covers failure cases.

Generalize the above to create new "RunCMake" test infrastructure that
can run CMake multiple times for a single project with different
variations and check for expected result/stdout/stderr.  Allow for both
successful and failing CMake project configuration cases.  This will be
useful to test error messages and failure behavior.

Tests/CMakeLists.txt

@@ -54,6 +54,7 @@ IF(BUILD_TESTING)
   ADD_SUBDIRECTORY(CMakeLib)
   ADD_SUBDIRECTORY(CMakeOnly)
   ADD_SUBDIRECTORY(CMakeCommands)
+  ADD_SUBDIRECTORY(RunCMake)
 
   ADD_SUBDIRECTORY(FindPackageModeMakefileTest)
 

Tests/README

@@ -16,10 +16,15 @@ your test to the test runs.
 This includes tests that will build something using try_compile() and friends,
 but nothing that expects add_executable(), add_library(), or add_test() to run.
 
-If this matches your test you should put it into the Tests/CMakeOnly/ directory.
+If the test configures the project only once and it must succeed then put it
-Create a subdirectory named like your test and write the CMakeLists.txt you
+into the Tests/CMakeOnly/ directory.  Create a subdirectory named like your
-need into that subdirectory. Use the add_CMakeOnly_test() macro from
+test and write the CMakeLists.txt you need into that subdirectory. Use the
-Tests/CMakeOnly/CMakeLists.txt to add your test to the test runs.
+add_CMakeOnly_test() macro from Tests/CMakeOnly/CMakeLists.txt to add your
+test to the test runs.
+
+If the test configures the project with multiple variations and verifies
+success or failure each time then put it into the Tests/RunCMake/ directory.
+Read the instructions in Tests/RunCMake/CMakeLists.txt to add a test.
 
 3. If you are testing something from the Modules directory
 

Tests/RunCMake/CMakeLists.txt

@@ -0,0 +1,41 @@
+# This directory contains tests that run CMake to configure a project
+# but do not actually build anything.  To add a test:
+#
+# 1.) Add a subdirectory named for the test.
+#
+# 2.) Call add_RunCMake_test and pass the test directory name.
+#
+# 3.) Create a RunCMakeTest.cmake script in the directory containing
+#   include(RunCMake)
+#   run_cmake(SubTest1)
+#   ...
+#   run_cmake(SubTestN)
+# where SubTest1..SubTestN are sub-test names each corresponding to
+# an independent CMake run and project configuration.
+#
+# 3.) Create a CMakeLists.txt file in the directory containing
+#   cmake_minimum_required(...)
+#   project(${RunCMake_TEST} NONE) # or languages needed
+#   include(${RunCMake_TEST}.cmake)
+# where "${RunCMake_TEST}" is literal.  A value for RunCMake_TEST
+# will be passed to CMake by the run_cmake macro when running each
+# sub-test.
+#
+# 4.) Create a <SubTest>.cmake file for each sub-test named above
+# containing the actual test code.  Optionally create files
+# containing expected test results:
+#   <SubTest>-result.txt = Process result expected if not "0"
+#   <SubTest>-stdout.txt = Regex matching expected stdout content
+#   <SubTest>-stderr.txt = Regex matching expected stderr content
+# Note that trailing newlines will be stripped from actual test
+# output before matching against the stdout and stderr expressions.
+
+macro(add_RunCMake_test test)
+  add_test(RunCMake.${test} ${CMAKE_CMAKE_COMMAND}
+    -DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
+    -DRunCMake_GENERATOR=${CMAKE_TEST_GENERATOR}
+    -DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
+    -DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${test}
+    -P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
+    )
+endmacro()

Tests/RunCMake/RunCMake.cmake

@@ -0,0 +1,69 @@
+foreach(arg
+    RunCMake_GENERATOR
+    RunCMake_SOURCE_DIR
+    RunCMake_BINARY_DIR
+    )
+  if(NOT DEFINED ${arg})
+    message(FATAL_ERROR "${arg} not given!")
+  endif()
+endforeach()
+
+function(run_cmake test)
+  set(top_src "${RunCMake_SOURCE_DIR}")
+  set(top_bin "${RunCMake_BINARY_DIR}")
+  if(EXISTS ${top_src}/${test}-result.txt)
+    file(READ ${top_src}/${test}-result.txt expect_result)
+    string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
+  else()
+    set(expect_result 0)
+  endif()
+  foreach(o out err)
+    if(EXISTS ${top_src}/${test}-std${o}.txt)
+      file(READ ${top_src}/${test}-std${o}.txt expect_std${o})
+      string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
+    else()
+      unset(expect_std${o})
+    endif()
+  endforeach()
+  set(source_dir "${top_src}")
+  set(binary_dir "${top_bin}/${test}-build")
+  file(REMOVE_RECURSE "${binary_dir}")
+  file(MAKE_DIRECTORY "${binary_dir}")
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} "${source_dir}"
+              -G "${RunCMake_GENERATOR}" -DRunCMake_TEST=${test}
+    WORKING_DIRECTORY "${binary_dir}"
+    OUTPUT_VARIABLE actual_stdout
+    ERROR_VARIABLE actual_stderr
+    RESULT_VARIABLE actual_result
+    )
+  set(msg "")
+  if(NOT "${actual_result}" STREQUAL "${expect_result}")
+    set(msg "${msg}Result is [${actual_result}], not [${expect_result}].\n")
+  endif()
+  foreach(o out err)
+    string(REGEX REPLACE "\n+$" "" actual_std${o} "${actual_std${o}}")
+    set(expect_${o} "")
+    if(DEFINED expect_std${o})
+      if(NOT "${actual_std${o}}" MATCHES "${expect_std${o}}")
+        string(REGEX REPLACE "\n" "\n expect-${o}> " expect_${o}
+          " expect-${o}> ${expect_std${o}}")
+        set(expect_${o} "Expected std${o} to match:\n${expect_${o}}\n")
+        set(msg "${msg}std${o} does not match that expected.\n")
+      endif()
+    endif()
+  endforeach()
+  if(msg)
+    string(REGEX REPLACE "\n" "\n actual-out> " actual_out " actual-out> ${actual_stdout}")
+    string(REGEX REPLACE "\n" "\n actual-err> " actual_err " actual-err> ${actual_stderr}")
+    message(SEND_ERROR "${test} - FAILED:\n"
+      "${msg}"
+      "${expect_out}"
+      "Actual stdout:\n${actual_out}\n"
+      "${expect_err}"
+      "Actual stderr:\n${actual_err}\n"
+      )
+  else()
+    message(STATUS "${test} - PASSED")
+  endif()
+endfunction()