Merge topic 'ctest_rerun_failed'

eb2decc ctest: Add --rerun-failed option

Source/CTest/cmCTestTestHandler.cxx

@@ -20,6 +20,7 @@
 #include <cmsys/Process.h>
 #include <cmsys/RegularExpression.hxx>
 #include <cmsys/Base64.h>
+#include <cmsys/Directory.hxx>
 #include "cmMakefile.h"
 #include "cmGlobalGenerator.h"
 #include "cmLocalGenerator.h"
@@ -537,6 +538,7 @@ int cmCTestTestHandler::ProcessHandler()
     this->UseExcludeRegExp();
     this->SetExcludeRegExp(val);
     }
+  this->SetRerunFailed(cmSystemTools::IsOn(this->GetOption("RerunFailed")));
 
   this->TestResults.clear();
 
@@ -819,6 +821,13 @@ void cmCTestTestHandler::ComputeTestList()
 {
   this->TestList.clear(); // clear list of test
   this->GetListOfTests();
+
+  if (this->RerunFailed)
+    {
+    this->ComputeTestListForRerunFailed();
+    return;
+    }
+
   cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
   // how many tests are in based on RegExp?
   int inREcnt = 0;
@@ -881,9 +890,47 @@ void cmCTestTestHandler::ComputeTestList()
   this->TotalNumberOfTests = this->TestList.size();
   // Set the TestList to the final list of all test
   this->TestList = finalList;
+
+  this->UpdateMaxTestNameWidth();
+}
+
+void cmCTestTestHandler::ComputeTestListForRerunFailed()
+{
+  this->ExpandTestsToRunInformationForRerunFailed();
+
+  cmCTestTestHandler::ListOfTests::iterator it;
+  ListOfTests finalList;
+  int cnt = 0;
+  for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
+    {
+    cnt ++;
+
+    // if this test is not in our list of tests to run, then skip it.
+    if ((this->TestsToRun.size() &&
+         std::find(this->TestsToRun.begin(), this->TestsToRun.end(), cnt)
+         == this->TestsToRun.end()))
+      {
+      continue;
+      }
+
+    it->Index = cnt;
+    finalList.push_back(*it);
+    }
+
+  // Save the total number of tests before exclusions
+  this->TotalNumberOfTests = this->TestList.size();
+
+  // Set the TestList to the list of failed tests to rerun
+  this->TestList = finalList;
+
+  this->UpdateMaxTestNameWidth();
+}
+
+void cmCTestTestHandler::UpdateMaxTestNameWidth()
+{
   std::string::size_type max = this->CTest->GetMaxTestNameWidth();
-  for (it = this->TestList.begin();
+  for ( cmCTestTestHandler::ListOfTests::iterator it = this->TestList.begin();
-       it != this->TestList.end(); it ++ )
+        it != this->TestList.end(); it ++ )
     {
     cmCTestTestProperties& p = *it;
     if(max < p.Name.size())
@@ -1708,6 +1755,91 @@ void cmCTestTestHandler::ExpandTestsToRunInformation(size_t numTests)
   this->TestsToRun.erase(new_end, this->TestsToRun.end());
 }
 
+void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
+{
+
+  std::string dirName = this->CTest->GetBinaryDir() + "/Testing/Temporary";
+
+  cmsys::Directory directory;
+  if (directory.Load(dirName.c_str()) == 0)
+    {
+    cmCTestLog(this->CTest, ERROR_MESSAGE, "Unable to read the contents of "
+      << dirName << std::endl);
+    return;
+    }
+
+  int numFiles = static_cast<int>
+    (cmsys::Directory::GetNumberOfFilesInDirectory(dirName.c_str()));
+  std::string pattern = "LastTestsFailed";
+  std::string logName = "";
+
+  for (int i = 0; i < numFiles; ++i)
+    {
+    std::string fileName = directory.GetFile(i);
+    // bcc crashes if we attempt a normal substring comparison,
+    // hence the following workaround
+    std::string fileNameSubstring = fileName.substr(0, pattern.length());
+    if (fileNameSubstring.compare(pattern) != 0)
+      {
+      continue;
+      }
+    if (logName == "")
+      {
+      logName = fileName;
+      }
+    else
+      {
+      // if multiple matching logs were found we use the most recently
+      // modified one.
+      int res;
+      cmSystemTools::FileTimeCompare(logName.c_str(), fileName.c_str(), &res);
+      if (res == -1)
+        {
+        logName = fileName;
+        }
+      }
+    }
+
+  std::string lastTestsFailedLog = this->CTest->GetBinaryDir()
+    + "/Testing/Temporary/" + logName;
+
+  if ( !cmSystemTools::FileExists(lastTestsFailedLog.c_str()) )
+    {
+    if ( !this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() )
+      {
+      cmCTestLog(this->CTest, ERROR_MESSAGE, lastTestsFailedLog
+        << " does not exist!" << std::endl);
+      }
+    return;
+    }
+
+  // parse the list of tests to rerun from LastTestsFailed.log
+  std::ifstream ifs(lastTestsFailedLog.c_str());
+  if ( ifs )
+    {
+    std::string line;
+    std::string::size_type pos;
+    while ( cmSystemTools::GetLineFromStream(ifs, line) )
+      {
+      pos = line.find(':', 0);
+      if (pos == line.npos)
+        {
+        continue;
+        }
+
+      int val = atoi(line.substr(0, pos).c_str());
+      this->TestsToRun.push_back(val);
+      }
+    ifs.close();
+    }
+  else if ( !this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() )
+    {
+    cmCTestLog(this->CTest, ERROR_MESSAGE, "Problem reading file: "
+      << lastTestsFailedLog.c_str() <<
+      " while generating list of previously failed tests." << std::endl);
+    }
+}
+
 //----------------------------------------------------------------------
 // Just for convenience
 #define SPACE_REGEX "[ \t\r\n]"

Source/CTest/cmCTestTestHandler.h

@@ -43,6 +43,12 @@ public:
    */
   void SetUseUnion(bool val) { this->UseUnion = val; }
 
+  /**
+   * Set whether or not CTest should only execute the tests that failed
+   * on the previous run.  By default this is false.
+   */
+  void SetRerunFailed(bool val) { this->RerunFailed = val; }
+
   /**
    * This method is called when reading CTest custom file
    */
@@ -213,6 +219,12 @@ private:
   // based on union regex and -I stuff
   void ComputeTestList();
 
+  // compute the lists of tests that will actually run
+  // based on LastTestFailed.log
+  void ComputeTestListForRerunFailed();
+
+  void UpdateMaxTestNameWidth();
+
   bool GetValue(const char* tag,
                 std::string& value,
                 std::ifstream& fin);
@@ -235,6 +247,7 @@ private:
 
   const char* GetTestStatus(int status);
   void ExpandTestsToRunInformation(size_t numPossibleTests);
+  void ExpandTestsToRunInformationForRerunFailed();
 
   std::vector<cmStdString> CustomPreTest;
   std::vector<cmStdString> CustomPostTest;
@@ -268,6 +281,8 @@ private:
   cmsys::RegularExpression DartStuff;
 
   std::ostream* LogFile;
+
+  bool RerunFailed;
 };
 
 #endif

Source/cmCTest.cxx

@@ -2187,6 +2187,12 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
     this->GetHandler("memcheck")->
       SetPersistentOption("ExcludeRegularExpression", args[i].c_str());
     }
+
+  if(this->CheckArgument(arg, "--rerun-failed"))
+    {
+    this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
+    this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
+    }
 }
 
 //----------------------------------------------------------------------

Source/ctest.cxx

@@ -150,6 +150,13 @@ static const char * cmDocumentationOptions[][3] =
   {"-U, --union", "Take the Union of -I and -R",
    "When both -R and -I are specified by default the intersection of "
    "tests are run. By specifying -U the union of tests is run instead."},
+  {"--rerun-failed", "Run only the tests that failed previously",
+   "This option tells ctest to perform only the tests that failed during its "
+   "previous run. When this option is specified, ctest ignores all other "
+   "options intended to modify the list of tests to run "
+   "(-L, -R, -E, -LE, -I, etc). In the event that CTest runs and no tests "
+   "fail, subsequent calls to ctest with the --rerun-failed option will "
+   "run the set of tests that most recently failed (if any)."},
   {"--max-width <width>", "Set the max width for a test name to output",
    "Set the maximum width for each test name to show in the output.  This "
    "allows the user to widen the output to avoid clipping the test name which "

Tests/CMakeLists.txt

@@ -2140,6 +2140,16 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
   set_tests_properties(CTestTestTimeout PROPERTIES
     PASS_REGULAR_EXPRESSION "TestTimeout *\\.+ *\\*\\*\\*Timeout.*CheckChild *\\.+ *Passed")
 
+  # this test only runs correctly if WORKING_DIRECTORY is honored.
+  if (NOT CMAKE_VERSION VERSION_LESS "2.8.4")
+    add_test(
+      NAME CTestTestRerunFailed
+      COMMAND ${CMAKE_CTEST_COMMAND} --rerun-failed)
+    set_tests_properties(CTestTestRerunFailed PROPERTIES
+      PASS_REGULAR_EXPRESSION "1/1 Test #1: TestTimeout" DEPENDS CTestTestTimeout
+      WORKING_DIRECTORY ${CMake_BINARY_DIR}/Tests/CTestTestTimeout)
+  endif ()
+
   configure_file(
     "${CMake_SOURCE_DIR}/Tests/CTestTestZeroTimeout/test.cmake.in"
     "${CMake_BINARY_DIR}/Tests/CTestTestZeroTimeout/test.cmake"