Hook for scheduling tests in a random order

This may help statistically detect implicit dependencies among unit
tests while running in parallel.
This commit is contained in:
Zach Mullen 2009-10-29 15:30:12 -04:00
parent e183581b14
commit 8612aa10b6
7 changed files with 44 additions and 5 deletions

View File

@ -24,6 +24,7 @@ cmCTestTestCommand::cmCTestTestCommand()
this->Arguments[ctt_EXCLUDE_LABEL] = "EXCLUDE_LABEL"; this->Arguments[ctt_EXCLUDE_LABEL] = "EXCLUDE_LABEL";
this->Arguments[ctt_INCLUDE_LABEL] = "INCLUDE_LABEL"; this->Arguments[ctt_INCLUDE_LABEL] = "INCLUDE_LABEL";
this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL"; this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL";
this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM";
this->Arguments[ctt_LAST] = 0; this->Arguments[ctt_LAST] = 0;
this->Last = ctt_LAST; this->Last = ctt_LAST;
} }
@ -91,6 +92,11 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
handler->SetOption("ParallelLevel", handler->SetOption("ParallelLevel",
this->Values[ctt_PARALLEL_LEVEL]); this->Values[ctt_PARALLEL_LEVEL]);
} }
if(this->Values[ctt_SCHEDULE_RANDOM])
{
handler->SetOption("ScheduleRandom",
this->Values[ctt_SCHEDULE_RANDOM]);
}
return handler; return handler;
} }

View File

@ -61,7 +61,8 @@ public:
" [INCLUDE include regex] [RETURN_VALUE res] \n" " [INCLUDE include regex] [RETURN_VALUE res] \n"
" [EXCLUDE_LABEL exclude regex] \n" " [EXCLUDE_LABEL exclude regex] \n"
" [INCLUDE_LABEL label regex] \n" " [INCLUDE_LABEL label regex] \n"
" [PARALLEL_LEVEL level]) \n" " [PARALLEL_LEVEL level] \n"
" [SCHEDULE_RANDOM on]) \n"
"Tests the given build directory and stores results in Test.xml. The " "Tests the given build directory and stores results in Test.xml. The "
"second argument is a variable that will hold value. Optionally, " "second argument is a variable that will hold value. Optionally, "
"you can specify the starting test number START, the ending test number " "you can specify the starting test number START, the ending test number "
@ -70,7 +71,9 @@ public:
"to not run EXCLUDE. EXCLUDE_LABEL and INCLUDE_LABEL are regular " "to not run EXCLUDE. EXCLUDE_LABEL and INCLUDE_LABEL are regular "
"expression for test to be included or excluded by the test " "expression for test to be included or excluded by the test "
"property LABEL. PARALLEL_LEVEL should be set to a positive number " "property LABEL. PARALLEL_LEVEL should be set to a positive number "
"representing the number of tests to be run in parallel." "representing the number of tests to be run in parallel. "
"SCHEDULE_RANDOM will launch tests in a random order, and is "
"typically used to detect implicit test dependencies."
"\n" "\n"
CTEST_COMMAND_APPEND_OPTION_DOCS; CTEST_COMMAND_APPEND_OPTION_DOCS;
} }
@ -92,6 +95,7 @@ protected:
ctt_EXCLUDE_LABEL, ctt_EXCLUDE_LABEL,
ctt_INCLUDE_LABEL, ctt_INCLUDE_LABEL,
ctt_PARALLEL_LEVEL, ctt_PARALLEL_LEVEL,
ctt_SCHEDULE_RANDOM,
ctt_LAST ctt_LAST
}; };
}; };

View File

@ -491,11 +491,16 @@ int cmCTestTestHandler::ProcessHandler()
{ {
// Update internal data structure from generic one // Update internal data structure from generic one
this->SetTestsToRunInformation(this->GetOption("TestsToRunInformation")); this->SetTestsToRunInformation(this->GetOption("TestsToRunInformation"));
this->SetUseUnion(cmSystemTools::IsOn(this->GetOption("UseUnion"))); this->SetUseUnion(cmSystemTools::IsOn(this->GetOption("UseUnion")));
if(cmSystemTools::IsOn(this->GetOption("ScheduleRandom")))
{
this->CTest->SetScheduleType("Random");
}
if(this->GetOption("ParallelLevel")) if(this->GetOption("ParallelLevel"))
{ {
this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel"))); this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
} }
const char* val; const char* val;
val = this->GetOption("LabelRegularExpression"); val = this->GetOption("LabelRegularExpression");
if ( val ) if ( val )
@ -1021,12 +1026,23 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
cmCTestMultiProcessHandler::TestMap tests; cmCTestMultiProcessHandler::TestMap tests;
cmCTestMultiProcessHandler::PropertiesMap properties; cmCTestMultiProcessHandler::PropertiesMap properties;
bool randomSchedule = this->CTest->GetScheduleType() == "Random";
if(randomSchedule)
{
srand((unsigned)time(0));
}
for (ListOfTests::iterator it = this->TestList.begin(); for (ListOfTests::iterator it = this->TestList.begin();
it != this->TestList.end(); ++it) it != this->TestList.end(); ++it)
{ {
cmCTestTestProperties& p = *it; cmCTestTestProperties& p = *it;
cmCTestMultiProcessHandler::TestSet depends; cmCTestMultiProcessHandler::TestSet depends;
if(randomSchedule)
{
p.Cost = rand();
}
if(p.Depends.size()) if(p.Depends.size())
{ {
for(std::vector<std::string>::iterator i = p.Depends.begin(); for(std::vector<std::string>::iterator i = p.Depends.begin();

View File

@ -225,6 +225,7 @@ cmCTest::cmCTest()
this->TimeOut = 0; this->TimeOut = 0;
this->CompressXMLFiles = false; this->CompressXMLFiles = false;
this->CTestConfigFile = ""; this->CTestConfigFile = "";
this->ScheduleType = "";
this->OutputLogFile = 0; this->OutputLogFile = 0;
this->OutputLogFileLastTag = -1; this->OutputLogFileLastTag = -1;
this->SuppressUpdatingCTestConfiguration = false; this->SuppressUpdatingCTestConfiguration = false;
@ -2027,6 +2028,11 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
cmakeAndTest = true; cmakeAndTest = true;
} }
if(this->CheckArgument(arg, "--schedule-random"))
{
this->ScheduleType = "Random";
}
// pass the argument to all the handlers as well, but i may no longer be // pass the argument to all the handlers as well, but i may no longer be
// set to what it was originally so I'm not sure this is working as // set to what it was originally so I'm not sure this is working as
// intended // intended

View File

@ -191,6 +191,9 @@ public:
///! Should we only show what we would do? ///! Should we only show what we would do?
bool GetShowOnly(); bool GetShowOnly();
//Used for parallel ctest job scheduling
std::string GetScheduleType() { return this->ScheduleType; }
void SetScheduleType(std::string type) { this->ScheduleType = type; }
///! The max output width ///! The max output width
int GetMaxTestNameWidth() const; int GetMaxTestNameWidth() const;
@ -374,6 +377,7 @@ public:
bool GetLabelSummary() { return this->LabelSummary;} bool GetLabelSummary() { return this->LabelSummary;}
private: private:
std::string ConfigType; std::string ConfigType;
std::string ScheduleType;
bool Verbose; bool Verbose;
bool ExtraVerbose; bool ExtraVerbose;
bool ProduceXML; bool ProduceXML;

View File

@ -208,6 +208,9 @@ static const char * cmDocumentationOptions[][3] =
"By default CTest will run child CTest instances within the same process. " "By default CTest will run child CTest instances within the same process. "
"If this behavior is not desired, this argument will enforce new " "If this behavior is not desired, this argument will enforce new "
"processes for child CTest processes." }, "processes for child CTest processes." },
{"--schedule-random", "Use a random order for scheduling tests",
"This option will run the tests in a random order. It is commonly used to "
"detect implicit dependencies in a test suite." },
{"--submit-index", "Submit individual dashboard tests with specific index", {"--submit-index", "Submit individual dashboard tests with specific index",
"This option allows performing the same CTest action (such as test) " "This option allows performing the same CTest action (such as test) "
"multiple times and submit all stages to the same dashboard (Dart2 " "multiple times and submit all stages to the same dashboard (Dart2 "

View File

@ -50,7 +50,7 @@ CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_READ_CUSTOM_FILES("${CTEST_BINARY_DIRECTORY}") CTEST_READ_CUSTOM_FILES("${CTEST_BINARY_DIRECTORY}")
CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_SUBMIT(RETURN_VALUE res) CTEST_SUBMIT(RETURN_VALUE res)
@ -97,7 +97,7 @@ IF(svncommand)
CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res) CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5) CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5) CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_SUBMIT(RETURN_VALUE res) CTEST_SUBMIT(RETURN_VALUE res)