Hook for scheduling tests in a random order

This may help statistically detect implicit dependencies among unit
tests while running in parallel.

Source/CTest/cmCTestTestCommand.cxx

@@ -24,6 +24,7 @@ cmCTestTestCommand::cmCTestTestCommand()
   this->Arguments[ctt_EXCLUDE_LABEL] = "EXCLUDE_LABEL";
   this->Arguments[ctt_INCLUDE_LABEL] = "INCLUDE_LABEL";
   this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL";
+  this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM";
   this->Arguments[ctt_LAST] = 0;
   this->Last = ctt_LAST;
 }
@@ -91,6 +92,11 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
     handler->SetOption("ParallelLevel",
                        this->Values[ctt_PARALLEL_LEVEL]);
     }
+  if(this->Values[ctt_SCHEDULE_RANDOM])
+    {
+    handler->SetOption("ScheduleRandom",
+                       this->Values[ctt_SCHEDULE_RANDOM]);
+    }
   return handler;
 }
 

Source/CTest/cmCTestTestCommand.h

@@ -61,7 +61,8 @@ public:
       "             [INCLUDE include regex] [RETURN_VALUE res] \n" 
       "             [EXCLUDE_LABEL exclude regex] \n"
       "             [INCLUDE_LABEL label regex] \n"
-      "             [PARALLEL_LEVEL level]) \n"
+      "             [PARALLEL_LEVEL level] \n"
+      "             [SCHEDULE_RANDOM on]) \n"
       "Tests the given build directory and stores results in Test.xml. The "
       "second argument is a variable that will hold value. Optionally, "
       "you can specify the starting test number START, the ending test number "
@@ -71,6 +72,8 @@ public:
       "expression for test to be included or excluded by the test "
       "property LABEL. PARALLEL_LEVEL should be set to a positive number "
       "representing the number of tests to be run in parallel. "
+      "SCHEDULE_RANDOM will launch tests in a random order, and is "
+      "typically used to detect implicit test dependencies."
       "\n"
       CTEST_COMMAND_APPEND_OPTION_DOCS;
     }
@@ -92,6 +95,7 @@ protected:
     ctt_EXCLUDE_LABEL,
     ctt_INCLUDE_LABEL,
     ctt_PARALLEL_LEVEL,
+    ctt_SCHEDULE_RANDOM,
     ctt_LAST
   };
 };

Source/CTest/cmCTestTestHandler.cxx

@@ -492,10 +492,15 @@ int cmCTestTestHandler::ProcessHandler()
   // Update internal data structure from generic one
   this->SetTestsToRunInformation(this->GetOption("TestsToRunInformation"));
   this->SetUseUnion(cmSystemTools::IsOn(this->GetOption("UseUnion")));
+  if(cmSystemTools::IsOn(this->GetOption("ScheduleRandom")))
+    {
+    this->CTest->SetScheduleType("Random");
+    }
   if(this->GetOption("ParallelLevel"))
     {
     this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
     }
+
   const char* val;
   val = this->GetOption("LabelRegularExpression");
   if ( val )
@@ -1021,12 +1026,23 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
   cmCTestMultiProcessHandler::TestMap tests;
   cmCTestMultiProcessHandler::PropertiesMap properties;
   
+  bool randomSchedule = this->CTest->GetScheduleType() == "Random";
+  if(randomSchedule)
+  {
+    srand((unsigned)time(0));
+  }
+
   for (ListOfTests::iterator it = this->TestList.begin();
        it != this->TestList.end(); ++it)
     { 
     cmCTestTestProperties& p = *it;
     cmCTestMultiProcessHandler::TestSet depends;
 
+    if(randomSchedule)
+      {
+      p.Cost = rand();
+      }
+
     if(p.Depends.size())
       {
       for(std::vector<std::string>::iterator i = p.Depends.begin();

Source/cmCTest.cxx

@@ -225,6 +225,7 @@ cmCTest::cmCTest()
   this->TimeOut                = 0;
   this->CompressXMLFiles       = false;
   this->CTestConfigFile        = "";
+  this->ScheduleType           = "";
   this->OutputLogFile          = 0;
   this->OutputLogFileLastTag   = -1;
   this->SuppressUpdatingCTestConfiguration = false;
@@ -2027,6 +2028,11 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
       cmakeAndTest = true;
       }
 
+    if(this->CheckArgument(arg, "--schedule-random"))
+      {
+      this->ScheduleType = "Random";
+      }
+
     // pass the argument to all the handlers as well, but i may no longer be
     // set to what it was originally so I'm not sure this is working as
     // intended

Source/cmCTest.h

@@ -191,6 +191,9 @@ public:
   ///! Should we only show what we would do?
   bool GetShowOnly();
 
+  //Used for parallel ctest job scheduling
+  std::string GetScheduleType() { return this->ScheduleType; }
+  void SetScheduleType(std::string type) { this->ScheduleType = type; }
 
   ///! The max output width
   int GetMaxTestNameWidth() const;
@@ -374,6 +377,7 @@ public:
   bool GetLabelSummary() { return this->LabelSummary;}
 private:
   std::string ConfigType;
+  std::string ScheduleType;
   bool Verbose;
   bool ExtraVerbose;
   bool ProduceXML;

Source/ctest.cxx

@@ -208,6 +208,9 @@ static const char * cmDocumentationOptions[][3] =
    "By default CTest will run child CTest instances within the same process. "
    "If this behavior is not desired, this argument will enforce new "
    "processes for child CTest processes." },
+  {"--schedule-random", "Use a random order for scheduling tests",
+   "This option will run the tests in a random order. It is commonly used to "
+   "detect implicit dependencies in a test suite." },
   {"--submit-index", "Submit individual dashboard tests with specific index",
    "This option allows performing the same CTest action (such as test) "
    "multiple times and submit all stages to the same dashboard (Dart2 "

Tests/CTestTest3/test.cmake.in

@@ -50,7 +50,7 @@ CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
 CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
 CTEST_READ_CUSTOM_FILES("${CTEST_BINARY_DIRECTORY}")
 CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
-CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
 CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
 CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
 CTEST_SUBMIT(RETURN_VALUE res)
@@ -97,7 +97,7 @@ IF(svncommand)
   CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
   CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
   CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
-  CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
+  CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
   CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
   CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
   CTEST_SUBMIT(RETURN_VALUE res)