ctest: Add --rerun-failed option

Add a new command line argument to ctest.  This allows users to
rerun tests that failed during the previous call to ctest.  This
is accomplished by analyzing the most recently modified file named
"^LastTestsFailed*" in the Testing/Temporary subdirectory of the
project's binary directory.
This commit is contained in:
Zack Galbreath 2013-09-24 14:20:38 -04:00 committed by Brad King
parent 475635ec0f
commit eb2decc02d
5 changed files with 172 additions and 2 deletions

View File

@ -20,6 +20,7 @@
#include <cmsys/Process.h> #include <cmsys/Process.h>
#include <cmsys/RegularExpression.hxx> #include <cmsys/RegularExpression.hxx>
#include <cmsys/Base64.h> #include <cmsys/Base64.h>
#include <cmsys/Directory.hxx>
#include "cmMakefile.h" #include "cmMakefile.h"
#include "cmGlobalGenerator.h" #include "cmGlobalGenerator.h"
#include "cmLocalGenerator.h" #include "cmLocalGenerator.h"
@ -537,6 +538,7 @@ int cmCTestTestHandler::ProcessHandler()
this->UseExcludeRegExp(); this->UseExcludeRegExp();
this->SetExcludeRegExp(val); this->SetExcludeRegExp(val);
} }
this->SetRerunFailed(cmSystemTools::IsOn(this->GetOption("RerunFailed")));
this->TestResults.clear(); this->TestResults.clear();
@ -819,6 +821,13 @@ void cmCTestTestHandler::ComputeTestList()
{ {
this->TestList.clear(); // clear list of test this->TestList.clear(); // clear list of test
this->GetListOfTests(); this->GetListOfTests();
if (this->RerunFailed)
{
this->ComputeTestListForRerunFailed();
return;
}
cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size(); cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
// how many tests are in based on RegExp? // how many tests are in based on RegExp?
int inREcnt = 0; int inREcnt = 0;
@ -881,8 +890,46 @@ void cmCTestTestHandler::ComputeTestList()
this->TotalNumberOfTests = this->TestList.size(); this->TotalNumberOfTests = this->TestList.size();
// Set the TestList to the final list of all test // Set the TestList to the final list of all test
this->TestList = finalList; this->TestList = finalList;
this->UpdateMaxTestNameWidth();
}
void cmCTestTestHandler::ComputeTestListForRerunFailed()
{
this->ExpandTestsToRunInformationForRerunFailed();
cmCTestTestHandler::ListOfTests::iterator it;
ListOfTests finalList;
int cnt = 0;
for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
{
cnt ++;
// if this test is not in our list of tests to run, then skip it.
if ((this->TestsToRun.size() &&
std::find(this->TestsToRun.begin(), this->TestsToRun.end(), cnt)
== this->TestsToRun.end()))
{
continue;
}
it->Index = cnt;
finalList.push_back(*it);
}
// Save the total number of tests before exclusions
this->TotalNumberOfTests = this->TestList.size();
// Set the TestList to the list of failed tests to rerun
this->TestList = finalList;
this->UpdateMaxTestNameWidth();
}
void cmCTestTestHandler::UpdateMaxTestNameWidth()
{
std::string::size_type max = this->CTest->GetMaxTestNameWidth(); std::string::size_type max = this->CTest->GetMaxTestNameWidth();
for (it = this->TestList.begin(); for ( cmCTestTestHandler::ListOfTests::iterator it = this->TestList.begin();
it != this->TestList.end(); it ++ ) it != this->TestList.end(); it ++ )
{ {
cmCTestTestProperties& p = *it; cmCTestTestProperties& p = *it;
@ -1708,6 +1755,91 @@ void cmCTestTestHandler::ExpandTestsToRunInformation(size_t numTests)
this->TestsToRun.erase(new_end, this->TestsToRun.end()); this->TestsToRun.erase(new_end, this->TestsToRun.end());
} }
void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
{
std::string dirName = this->CTest->GetBinaryDir() + "/Testing/Temporary";
cmsys::Directory directory;
if (directory.Load(dirName.c_str()) == 0)
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Unable to read the contents of "
<< dirName << std::endl);
return;
}
int numFiles = static_cast<int>
(cmsys::Directory::GetNumberOfFilesInDirectory(dirName.c_str()));
std::string pattern = "LastTestsFailed";
std::string logName = "";
for (int i = 0; i < numFiles; ++i)
{
std::string fileName = directory.GetFile(i);
// bcc crashes if we attempt a normal substring comparison,
// hence the following workaround
std::string fileNameSubstring = fileName.substr(0, pattern.length());
if (fileNameSubstring.compare(pattern) != 0)
{
continue;
}
if (logName == "")
{
logName = fileName;
}
else
{
// if multiple matching logs were found we use the most recently
// modified one.
int res;
cmSystemTools::FileTimeCompare(logName.c_str(), fileName.c_str(), &res);
if (res == -1)
{
logName = fileName;
}
}
}
std::string lastTestsFailedLog = this->CTest->GetBinaryDir()
+ "/Testing/Temporary/" + logName;
if ( !cmSystemTools::FileExists(lastTestsFailedLog.c_str()) )
{
if ( !this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() )
{
cmCTestLog(this->CTest, ERROR_MESSAGE, lastTestsFailedLog
<< " does not exist!" << std::endl);
}
return;
}
// parse the list of tests to rerun from LastTestsFailed.log
std::ifstream ifs(lastTestsFailedLog.c_str());
if ( ifs )
{
std::string line;
std::string::size_type pos;
while ( cmSystemTools::GetLineFromStream(ifs, line) )
{
pos = line.find(':', 0);
if (pos == line.npos)
{
continue;
}
int val = atoi(line.substr(0, pos).c_str());
this->TestsToRun.push_back(val);
}
ifs.close();
}
else if ( !this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() )
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Problem reading file: "
<< lastTestsFailedLog.c_str() <<
" while generating list of previously failed tests." << std::endl);
}
}
//---------------------------------------------------------------------- //----------------------------------------------------------------------
// Just for convenience // Just for convenience
#define SPACE_REGEX "[ \t\r\n]" #define SPACE_REGEX "[ \t\r\n]"

View File

@ -43,6 +43,12 @@ public:
*/ */
void SetUseUnion(bool val) { this->UseUnion = val; } void SetUseUnion(bool val) { this->UseUnion = val; }
/**
* Set whether or not CTest should only execute the tests that failed
* on the previous run. By default this is false.
*/
void SetRerunFailed(bool val) { this->RerunFailed = val; }
/** /**
* This method is called when reading CTest custom file * This method is called when reading CTest custom file
*/ */
@ -213,6 +219,12 @@ private:
// based on union regex and -I stuff // based on union regex and -I stuff
void ComputeTestList(); void ComputeTestList();
// compute the lists of tests that will actually run
// based on LastTestFailed.log
void ComputeTestListForRerunFailed();
void UpdateMaxTestNameWidth();
bool GetValue(const char* tag, bool GetValue(const char* tag,
std::string& value, std::string& value,
std::ifstream& fin); std::ifstream& fin);
@ -235,6 +247,7 @@ private:
const char* GetTestStatus(int status); const char* GetTestStatus(int status);
void ExpandTestsToRunInformation(size_t numPossibleTests); void ExpandTestsToRunInformation(size_t numPossibleTests);
void ExpandTestsToRunInformationForRerunFailed();
std::vector<cmStdString> CustomPreTest; std::vector<cmStdString> CustomPreTest;
std::vector<cmStdString> CustomPostTest; std::vector<cmStdString> CustomPostTest;
@ -268,6 +281,8 @@ private:
cmsys::RegularExpression DartStuff; cmsys::RegularExpression DartStuff;
std::ostream* LogFile; std::ostream* LogFile;
bool RerunFailed;
}; };
#endif #endif

View File

@ -2187,6 +2187,12 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->GetHandler("memcheck")-> this->GetHandler("memcheck")->
SetPersistentOption("ExcludeRegularExpression", args[i].c_str()); SetPersistentOption("ExcludeRegularExpression", args[i].c_str());
} }
if(this->CheckArgument(arg, "--rerun-failed"))
{
this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
}
} }
//---------------------------------------------------------------------- //----------------------------------------------------------------------

View File

@ -150,6 +150,13 @@ static const char * cmDocumentationOptions[][3] =
{"-U, --union", "Take the Union of -I and -R", {"-U, --union", "Take the Union of -I and -R",
"When both -R and -I are specified by default the intersection of " "When both -R and -I are specified by default the intersection of "
"tests are run. By specifying -U the union of tests is run instead."}, "tests are run. By specifying -U the union of tests is run instead."},
{"--rerun-failed", "Run only the tests that failed previously",
"This option tells ctest to perform only the tests that failed during its "
"previous run. When this option is specified, ctest ignores all other "
"options intended to modify the list of tests to run "
"(-L, -R, -E, -LE, -I, etc). In the event that CTest runs and no tests "
"fail, subsequent calls to ctest with the --rerun-failed option will "
"run the set of tests that most recently failed (if any)."},
{"--max-width <width>", "Set the max width for a test name to output", {"--max-width <width>", "Set the max width for a test name to output",
"Set the maximum width for each test name to show in the output. This " "Set the maximum width for each test name to show in the output. This "
"allows the user to widen the output to avoid clipping the test name which " "allows the user to widen the output to avoid clipping the test name which "

View File

@ -2139,6 +2139,16 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
set_tests_properties(CTestTestTimeout PROPERTIES set_tests_properties(CTestTestTimeout PROPERTIES
PASS_REGULAR_EXPRESSION "TestTimeout *\\.+ *\\*\\*\\*Timeout.*CheckChild *\\.+ *Passed") PASS_REGULAR_EXPRESSION "TestTimeout *\\.+ *\\*\\*\\*Timeout.*CheckChild *\\.+ *Passed")
# this test only runs correctly if WORKING_DIRECTORY is honored.
if (NOT CMAKE_VERSION VERSION_LESS "2.8.4")
add_test(
NAME CTestTestRerunFailed
COMMAND ${CMAKE_CTEST_COMMAND} --rerun-failed)
set_tests_properties(CTestTestRerunFailed PROPERTIES
PASS_REGULAR_EXPRESSION "1/1 Test #1: TestTimeout" DEPENDS CTestTestTimeout
WORKING_DIRECTORY ${CMake_BINARY_DIR}/Tests/CTestTestTimeout)
endif ()
configure_file( configure_file(
"${CMake_SOURCE_DIR}/Tests/CTestTestZeroTimeout/test.cmake.in" "${CMake_SOURCE_DIR}/Tests/CTestTestZeroTimeout/test.cmake.in"
"${CMake_BINARY_DIR}/Tests/CTestTestZeroTimeout/test.cmake" "${CMake_BINARY_DIR}/Tests/CTestTestZeroTimeout/test.cmake"