allow to mark a test as "Not Run" with a specific return code (#8466)

Help/manual/cmake-properties.7.rst

@@ -247,6 +247,7 @@ Properties on Tests
    /prop_test/REQUIRED_FILES
    /prop_test/RESOURCE_LOCK
    /prop_test/RUN_SERIAL
+   /prop_test/SKIP_RETURN_CODE
    /prop_test/TIMEOUT
    /prop_test/WILL_FAIL
    /prop_test/WORKING_DIRECTORY

Help/prop_test/SKIP_RETURN_CODE.rst

@@ -0,0 +1,9 @@
+SKIP_RETURN_CODE
+----------------
+
+Return code to mark a test as skipped.
+
+Sometimes only a test itself can determine if all requirements for the
+test are met. If such a situation should not be considered a hard failure
+a return code of the process can be specified that will mark the test as
+"Not Run" if it is encountered.

Source/CTest/cmCTestRunTest.cxx

@@ -206,7 +206,13 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     bool success =
       !forceFail &&  (retVal == 0 ||
       this->TestProperties->RequiredRegularExpressions.size());
-    if((success && !this->TestProperties->WillFail)
+    if(this->TestProperties->SkipReturnCode >= 0
+      && this->TestProperties->SkipReturnCode == retVal)
+      {
+      this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped ");
+      }
+    else if((success && !this->TestProperties->WillFail)
       || (!success && this->TestProperties->WillFail))
       {
       this->TestResult.Status = cmCTestTestHandler::COMPLETED;

Source/CTest/cmCTestTestHandler.cxx

@@ -2229,6 +2229,14 @@ bool cmCTestTestHandler::SetTestsProperties(
               rtit->Processors = 1;
               }
             }
+          if ( key == "SKIP_RETURN_CODE" )
+            {
+            rtit->SkipReturnCode = atoi(val.c_str());
+            if(rtit->SkipReturnCode < 0 || rtit->SkipReturnCode > 255)
+              {
+              rtit->SkipReturnCode = -1;
+              }
+            }
           if ( key == "DEPENDS" )
             {
             std::vector<std::string> lval;
@@ -2364,6 +2372,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args)
   test.ExplicitTimeout = false;
   test.Cost = 0;
   test.Processors = 1;
+  test.SkipReturnCode = -1;
   test.PreviousRuns = 0;
   if (this->UseIncludeRegExpFlag &&
     !this->IncludeTestsRegularExpression.find(testname.c_str()))

Source/CTest/cmCTestTestHandler.h

@@ -109,6 +109,8 @@ public:
     int Index;
     //Requested number of process slots
     int Processors;
+    // return code of test which will mark test as "not run"
+    int SkipReturnCode;
     std::vector<std::string> Environment;
     std::vector<std::string> Labels;
     std::set<std::string> LockedResources;

Tests/CMakeLists.txt

@@ -2230,6 +2230,18 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
     --output-log "${CMake_BINARY_DIR}/Tests/CTestTestParallel/testOutput.log"
     )
 
+  configure_file(
+    "${CMake_SOURCE_DIR}/Tests/CTestTestSkipReturnCode/test.cmake.in"
+    "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/test.cmake"
+    @ONLY ESCAPE_QUOTES)
+  add_test(CTestTestSkipReturnCode ${CMAKE_CTEST_COMMAND}
+    -S "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/test.cmake" -V
+    --output-log "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/testOutput.log"
+    -C \${CTEST_CONFIGURATION_TYPE}
+    )
+  set_tests_properties(CTestTestSkipReturnCode PROPERTIES
+    PASS_REGULAR_EXPRESSION "CMakeV1 \\.* +Passed.*CMakeV2 \\.+\\*+Skipped")
+
   ADD_TEST_MACRO(CTestTestSerialInDepends ${CMAKE_CTEST_COMMAND} -j 4
     --output-on-failure -C "\${CTestTest_CONFIG}")
 

Tests/CTestTestSkipReturnCode/CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 2.8.12)
+project(CTestTestSkipReturnCode)
+include(CTest)
+
+add_test (NAME CMakeV1 COMMAND ${CMAKE_COMMAND} "--version")
+add_test (NAME CMakeV2 COMMAND ${CMAKE_COMMAND} "--version")
+
+set_tests_properties(CMakeV2 PROPERTIES SKIP_RETURN_CODE 0)

Tests/CTestTestSkipReturnCode/CTestConfig.cmake

@@ -0,0 +1,7 @@
+set (CTEST_PROJECT_NAME "CTestTestSkipReturnCode")
+set (CTEST_NIGHTLY_START_TIME "21:00:00 EDT")
+set (CTEST_DART_SERVER_VERSION "2")
+set(CTEST_DROP_METHOD "http")
+set(CTEST_DROP_SITE "www.cdash.org")
+set(CTEST_DROP_LOCATION "/CDash/submit.php?project=PublicDashboard")
+set(CTEST_DROP_SITE_CDASH TRUE)

Tests/CTestTestSkipReturnCode/test.cmake.in

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 2.4)
+
+# Settings:
+set(CTEST_DASHBOARD_ROOT                "@CMake_BINARY_DIR@/Tests/CTestTest")
+set(CTEST_SITE                          "@SITE@")
+set(CTEST_BUILD_NAME                    "CTestTest-@BUILDNAME@-SkipReturnCode")
+
+set(CTEST_SOURCE_DIRECTORY              "@CMake_SOURCE_DIR@/Tests/CTestTestSkipReturnCode")
+set(CTEST_BINARY_DIRECTORY              "@CMake_BINARY_DIR@/Tests/CTestTestSkipReturnCode")
+set(CTEST_CVS_COMMAND                   "@CVSCOMMAND@")
+set(CTEST_CMAKE_GENERATOR               "@CMAKE_TEST_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@CMAKE_TEST_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_COVERAGE_COMMAND              "@COVERAGE_COMMAND@")
+set(CTEST_NOTES_FILES                   "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}")
+
+#CTEST_EMPTY_BINARY_DIRECTORY(${CTEST_BINARY_DIRECTORY})
+
+CTEST_START(Experimental)
+CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)