Merge topic 'negative-test-costs'

6b0c7de Support explicitly set test costs in non-parallel testing.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -428,7 +428,10 @@ void cmCTestMultiProcessHandler::ReadCostData()
       if(index == -1) continue;
 
       this->Properties[index]->PreviousRuns = prev;
-      if(this->Properties[index] && this->Properties[index]->Cost == 0)
+      // When not running in parallel mode, don't use cost data
+      if(this->ParallelLevel > 1 &&
+         this->Properties[index] &&
+         this->Properties[index]->Cost == 0)
         {
         this->Properties[index]->Cost = cost;
         }
@@ -469,20 +472,19 @@ void cmCTestMultiProcessHandler::CreateTestCostList()
     {
     SortedTests.push_back(i->first);
 
-    //If the test failed last time, it should be run first, so max the cost
+    //If the test failed last time, it should be run first, so max the cost.
-    if(std::find(this->LastTestsFailed.begin(),
+    //Only do this for parallel runs; in non-parallel runs, avoid clobbering
-                 this->LastTestsFailed.end(),
+    //the test's explicitly set cost.
-                 this->Properties[i->first]->Name)
+    if(this->ParallelLevel > 1 &&
-       != this->LastTestsFailed.end())
+       std::find(this->LastTestsFailed.begin(), this->LastTestsFailed.end(),
+       this->Properties[i->first]->Name) != this->LastTestsFailed.end())
       {
       this->Properties[i->first]->Cost = FLT_MAX;
       }
     }
-  if(this->ParallelLevel > 1)
+
-    {
+  TestComparator comp(this);
-    TestComparator comp(this);
+  std::sort(SortedTests.begin(), SortedTests.end(), comp);
-    std::sort(SortedTests.begin(), SortedTests.end(), comp);
-    }
 }
 
 //---------------------------------------------------------

Tests/CMakeLists.txt

@@ -1405,7 +1405,20 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
     --output-log "${CMake_BINARY_DIR}/Tests/CTestTestScheduler/testOutput.log"
     )
   SET_TESTS_PROPERTIES(CTestTestScheduler PROPERTIES
-    PASS_REGULAR_EXPRESSION "Start 1.*Start 2.*Start 3.*Start 4.*Start 4.*Start 3.*Start 2.*Start 1")
+    PASS_REGULAR_EXPRESSION "Start 1.*Start 2.*Start 3.*Start 4.*Start 4.*Start 3.*Start 2.*Start 1"
+    RESOURCE_LOCK "CostData")
+
+  CONFIGURE_FILE(
+    "${CMake_SOURCE_DIR}/Tests/CTestTestCostSerial/test.cmake.in"
+    "${CMake_BINARY_DIR}/Tests/CTestTestCostSerial/test.cmake"
+    @ONLY ESCAPE_QUOTES)
+  ADD_TEST(CTestTestCostSerial ${CMAKE_CTEST_COMMAND}
+    -S "${CMake_BINARY_DIR}/Tests/CTestTestCostSerial/test.cmake" -V
+    --output-log "${CMake_BINARY_DIR}/Tests/CTestTestCostSerial/testOutput.log"
+    )
+  SET_TESTS_PROPERTIES(CTestTestCostSerial PROPERTIES
+    PASS_REGULAR_EXPRESSION "Start 2.*Start 3.*Start 1.*Start 2.*Start 3.*Start 1"
+    RESOURCE_LOCK "CostData")
 
   CONFIGURE_FILE(
     "${CMake_SOURCE_DIR}/Tests/CTestTestStopTime/test.cmake.in"

Tests/CTestTestCostSerial/CMakeLists.txt

@@ -0,0 +1,13 @@
+CMAKE_MINIMUM_REQUIRED (VERSION 2.6)
+PROJECT (CTestTestCostSerial)
+INCLUDE (CTest)
+
+ADD_EXECUTABLE (Sleep sleep.c)
+
+FOREACH (index RANGE 1 3)
+  ADD_TEST (TestSleep${index} Sleep)
+ENDFOREACH (index RANGE 1 3)
+
+SET_TESTS_PROPERTIES(TestSleep1 PROPERTIES COST -500)
+SET_TESTS_PROPERTIES(TestSleep2 PROPERTIES COST 12)
+SET_TESTS_PROPERTIES(TestSleep3 PROPERTIES COST 0)

Tests/CTestTestCostSerial/CTestConfig.cmake

@@ -0,0 +1,7 @@
+set(CTEST_PROJECT_NAME "CTestTestCostSerial")
+set(CTEST_NIGHTLY_START_TIME "21:00:00 EDT")
+set(CTEST_DART_SERVER_VERSION "2")
+set(CTEST_DROP_METHOD "http")
+set(CTEST_DROP_SITE "www.cdash.org")
+set(CTEST_DROP_LOCATION "/CDash/submit.php?project=PublicDashboard")
+set(CTEST_DROP_SITE_CDASH TRUE)

Tests/CTestTestCostSerial/sleep.c

@@ -0,0 +1,16 @@
+#if defined(_WIN32)
+# include <windows.h>
+#else
+# include <unistd.h>
+#endif
+
+/* sleeps for 1 second */
+int main(int argc, char** argv)
+{
+#if defined(_WIN32)
+  Sleep(1000);
+#else
+  sleep(1);
+#endif
+  return 0;
+}

Tests/CTestTestCostSerial/test.cmake.in

@@ -0,0 +1,31 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.1)
+
+# Settings:
+SET(CTEST_DASHBOARD_ROOT                "@CMake_BINARY_DIR@/Tests/CTestTest")
+SET(CTEST_SITE                          "@SITE@")
+SET(CTEST_BUILD_NAME                    "CTestTest-@BUILDNAME@-CostSerial")
+
+SET(CTEST_SOURCE_DIRECTORY              "@CMake_SOURCE_DIR@/Tests/CTestTestCostSerial")
+SET(CTEST_BINARY_DIRECTORY              "@CMake_BINARY_DIR@/Tests/CTestTestCostSerial")
+SET(CTEST_CVS_COMMAND                   "@CVSCOMMAND@")
+SET(CTEST_CMAKE_GENERATOR               "@CMAKE_TEST_GENERATOR@")
+SET(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+SET(CTEST_MEMORYCHECK_COMMAND           "@MEMORYCHECK_COMMAND@")
+SET(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "@MEMORYCHECK_SUPPRESSIONS_FILE@")
+SET(CTEST_MEMORYCHECK_COMMAND_OPTIONS   "@MEMORYCHECK_COMMAND_OPTIONS@")
+SET(CTEST_COVERAGE_COMMAND              "@COVERAGE_COMMAND@")
+SET(CTEST_NOTES_FILES                   "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}")
+
+#CTEST_EMPTY_BINARY_DIRECTORY(${CTEST_BINARY_DIRECTORY})
+
+# Remove old cost data file if it exists
+IF(EXISTS "${CTEST_BINARY_DIRECTORY}/Testing/Temporary/CTestCostData.txt")
+  FILE(REMOVE "${CTEST_BINARY_DIRECTORY}/Testing/Temporary/CTestCostData.txt")
+ENDIF(EXISTS "${CTEST_BINARY_DIRECTORY}/Testing/Temporary/CTestCostData.txt")
+
+CTEST_START(Experimental)
+CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+# Run test set a second time to make sure they run in same specified order
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)