ctest: Add a new --repeat-until-fail option

This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.

Help/manual/ctest.1.rst

@@ -194,6 +194,11 @@ Options
  subsequent calls to ctest with the --rerun-failed option will run
  the set of tests that most recently failed (if any).
 
+``--repeat-until-fail <n>``
+ Require each test to run ``<n>`` times without failing in order to pass.
+
+ This is useful in finding sporadic failures in test cases.
+
 ``--max-width <width>``
  Set the max width for a test name to output
 

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
+  if(this->CTest->GetRepeatUntilFail())
+    {
+    testRun->SetRunUntilFailOn();
+    testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+    }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
 
@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
     cmCTestRunTest* p = *i;
     int test = p->GetIndex();
 
-    if(p->EndTest(this->Completed, this->Total, true))
+    bool testResult = p->EndTest(this->Completed, this->Total, true);
+    if(p->StartAgain())
+      {
+      this->Completed--; // remove the completed test because run again
+      continue;
+      }
+    if(testResult)
       {
       this->Passed->push_back(p->GetTestProperties()->Name);
       }

Source/CTest/cmCTestRunTest.cxx

@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
   this->CompressedOutput = "";
   this->CompressionRatio = 2;
   this->StopTimePassed = false;
+  this->NumberOfRunsLeft = 1; // default to 1 run of the test
+  this->RunUntilFail = false; // default to run the test once
+  this->RunAgain = false;   // default to not having to run again
 }
 
 cmCTestRunTest::~cmCTestRunTest()
@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     this->MemCheckPostProcess();
     this->ComputeWeightedCost();
     }
-  // Always push the current TestResult onto the
+  // If the test does not need to rerun push the current TestResult onto the
   // TestHandler vector
-  this->TestHandler->TestResults.push_back(this->TestResult);
+  if(!this->NeedsToRerun())
+    {
+    this->TestHandler->TestResults.push_back(this->TestResult);
+    }
   delete this->TestProcess;
   return passed;
 }
 
+bool cmCTestRunTest::StartAgain()
+{
+  if(!this->RunAgain)
+    {
+    return false;
+    }
+  this->RunAgain = false; // reset
+  // change to tests directory
+  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+  cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
+  this->StartTest(this->TotalNumberOfTests);
+  // change back
+  cmSystemTools::ChangeDirectory(current_dir);
+  return true;
+}
+
+bool cmCTestRunTest::NeedsToRerun()
+{
+  this->NumberOfRunsLeft--;
+  if(this->NumberOfRunsLeft == 0)
+    {
+    return false;
+    }
+  // if number of runs left is not 0, and we are running until
+  // we find a failed test, then return true so the test can be
+  // restarted
+  if(this->RunUntilFail
+     && this->TestResult.Status == cmCTestTestHandler::COMPLETED)
+    {
+    this->RunAgain = true;
+    return true;
+    }
+  return false;
+}
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeWeightedCost()
 {
@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
 // Starts the execution of a test.  Returns once it has started
 bool cmCTestRunTest::StartTest(size_t total)
 {
+  this->TotalNumberOfTests = total; // save for rerun case
   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
     << "Start "
     << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeArguments()
 {
+  this->Arguments.clear(); // reset becaue this might be a rerun
   std::vector<std::string>::const_iterator j =
     this->TestProperties->Args.begin();
   ++j; // skip test name
-
   // find the test executable
   if(this->TestHandler->MemCheck)
     {
@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout,
 
 void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
 {
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << completed << "/");
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << total << " ");
+  // if this is the last or only run of this test
+  // then print out completed / total
+  // Only issue is if a test fails and we are running until fail
+  // then it will never print out the completed / total, same would
+  // got for run until pass.  Trick is when this is called we don't
+  // yet know if we are passing or failing.
+  if(this->NumberOfRunsLeft == 1)
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << completed << "/");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << total << " ");
+    }
+  // if this is one of several runs of a test just print blank space
+  // to keep things neat
+  else
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    }
 
   if ( this->TestHandler->MemCheck )
     {

Source/CTest/cmCTestRunTest.h

@@ -27,6 +27,8 @@ public:
   cmCTestRunTest(cmCTestTestHandler* handler);
   ~cmCTestRunTest();
 
+  void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
+  void SetRunUntilFailOn() { this->RunUntilFail = true;}
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
   { this->TestProperties = prop; }
 
@@ -58,7 +60,10 @@ public:
   void ComputeArguments();
 
   void ComputeWeightedCost();
+
+  bool StartAgain();
 private:
+  bool NeedsToRerun();
   void DartProcessing();
   void ExeNotFound(std::string exe);
   // Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
@@ -92,6 +97,10 @@ private:
   std::string ActualCommand;
   std::vector<std::string> Arguments;
   bool StopTimePassed;
+  bool RunUntilFail;
+  int NumberOfRunsLeft;
+  bool RunAgain;
+  size_t TotalNumberOfTests;
 };
 
 inline int getNumWidth(size_t n)

Source/cmCTest.cxx

@@ -328,6 +328,8 @@ cmCTest::cmCTest()
   this->OutputTestOutputOnTestFailure = false;
   this->ComputedCompressTestOutput = false;
   this->ComputedCompressMemCheckOutput = false;
+  this->RepeatTests = 1; // default to run each test once
+  this->RepeatUntilFail = false;
   if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE"))
     {
     this->OutputTestOutputOnTestFailure = true;
@@ -1983,11 +1985,11 @@ bool cmCTest::CheckArgument(const std::string& arg, const char* varg1,
 //----------------------------------------------------------------------
 // Processes one command line argument (and its arguments if any)
 // for many simple options and then returns
-void cmCTest::HandleCommandLineArguments(size_t &i,
-                                         std::vector<std::string> &args)
+bool cmCTest::HandleCommandLineArguments(size_t &i,
+                                         std::vector<std::string> &args,
+                                         std::string& errormsg)
 {
   std::string arg = args[i];
-
   if(this->CheckArgument(arg, "-F"))
     {
     this->Failover = true;
@@ -2005,6 +2007,27 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
     this->SetParallelLevel(plevel);
     this->ParallelLevelSetInCli = true;
     }
+  if(this->CheckArgument(arg, "--repeat-until-fail"))
+    {
+    if( i >= args.size() - 1)
+      {
+      errormsg = "'--repeat-until-fail' requires an argument";
+      return false;
+      }
+    i++;
+    long repeat = 1;
+    if(!cmSystemTools::StringToLong(args[i].c_str(), &repeat))
+      {
+      errormsg = "'--repeat-until-fail' given non-integer value '"
+        + args[i] + "'";
+      return false;
+      }
+    this->RepeatTests = static_cast<int>(repeat);
+    if(repeat > 1)
+      {
+      this->RepeatUntilFail = true;
+      }
+    }
 
   if(this->CheckArgument(arg, "--no-compress-output"))
     {
@@ -2190,6 +2213,7 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
     this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
     this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
     }
+  return true;
 }
 
 //----------------------------------------------------------------------
@@ -2272,7 +2296,12 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
   for(size_t i=1; i < args.size(); ++i)
     {
     // handle the simple commandline arguments
-    this->HandleCommandLineArguments(i,args);
+    std::string errormsg;
+    if(!this->HandleCommandLineArguments(i,args, errormsg))
+      {
+      cmSystemTools::Error(errormsg.c_str());
+      return 1;
+      }
 
     // handle the script arguments -S -SR -SP
     this->HandleScriptArguments(i,args,SRArgumentSpecified);

Source/cmCTest.h

@@ -429,8 +429,13 @@ public:
     {
     return this->Definitions;
     }
-
+  // return the number of times a test should be run
+  int GetTestRepeat() { return this->RepeatTests;}
+  // return true if test should run until fail
+  bool GetRepeatUntilFail() { return this->RepeatUntilFail;}
 private:
+  int RepeatTests;
+  bool RepeatUntilFail;
   std::string ConfigType;
   std::string ScheduleType;
   std::string StopTime;
@@ -535,8 +540,9 @@ private:
   bool AddVariableDefinition(const std::string &arg);
 
   //! parse and process most common command line arguments
-  void HandleCommandLineArguments(size_t &i,
-                                  std::vector<std::string> &args);
+  bool HandleCommandLineArguments(size_t &i,
+                                  std::vector<std::string> &args,
+                                  std::string& errormsg);
 
   //! hande the -S -SP and -SR arguments
   void HandleScriptArguments(size_t &i,

Source/ctest.cxx

@@ -75,6 +75,8 @@ static const char * cmDocumentationOptions[][2] =
    "Run a specific number of tests by number."},
   {"-U, --union", "Take the Union of -I and -R"},
   {"--rerun-failed", "Run only the tests that failed previously"},
+  {"--repeat-until-fail <n>", "Require each test to run <n> "
+   "times without failing in order to pass"},
   {"--max-width <width>", "Set the max width for a test name to output"},
   {"--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1."},
   {"--no-label-summary", "Disable timing summary information for labels."},

Tests/RunCMake/CMakeLists.txt

@@ -198,6 +198,7 @@ add_RunCMake_test(CommandLine)
 add_RunCMake_test(install)
 add_RunCMake_test(CPackInstallProperties)
 add_RunCMake_test(ExternalProject)
+add_RunCMake_test(CTestCommandLine)
 
 set(IfacePaths_INCLUDE_DIRECTORIES_ARGS -DTEST_PROP=INCLUDE_DIRECTORIES)
 add_RunCMake_test(IfacePaths_INCLUDE_DIRECTORIES TEST_DIR IfacePaths)

Tests/RunCMake/CTestCommandLine/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.0)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -0,0 +1,25 @@
+include(RunCMake)
+
+run_cmake_command(repeat-until-fail-bad1
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail
+  )
+run_cmake_command(repeat-until-fail-bad2
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail foo
+  )
+run_cmake_command(repeat-until-fail-good
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
+  )
+
+function(run_repeat_until_fail_tests)
+  # Use a single build tree for a few tests without cleaning.
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+
+  run_cmake(repeat-until-fail-cmake)
+  run_cmake_command(repeat-until-fail-ctest
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
+    )
+endfunction()
+run_repeat_until_fail_tests()

Tests/RunCMake/CTestCommandLine/init.cmake

@@ -0,0 +1,3 @@
+# This is run by test initialization in repeat-until-fail-cmake.cmake
+# with cmake -P.  It creates TEST_OUTPUT_FILE with a 0 in it.
+file(WRITE "${TEST_OUTPUT_FILE}" "0")

Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-fail' requires an argument$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-fail' given non-integer value 'foo'$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-cmake.cmake

@@ -0,0 +1,15 @@
+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/test1.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)

Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stderr.txt

@@ -0,0 +1 @@
+^Errors while running CTest$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stdout.txt

@@ -0,0 +1,30 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-fail-build
+    Start 1: initialization
+    Test #1: initialization ...................   Passed    [0-9.]+ sec
+    Start 1: initialization
+    Test #1: initialization ...................   Passed    [0-9.]+ sec
+    Start 1: initialization
+1/4 Test #1: initialization ...................   Passed    [0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................   Passed    [0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................\*\*\*Failed    [0-9.]+ sec
+    Start 3: hello
+    Test #3: hello ............................   Passed    [0-9.]+ sec
+    Start 3: hello
+    Test #3: hello ............................   Passed    [0-9.]+ sec
+    Start 3: hello
+3/4 Test #3: hello ............................   Passed    [0-9.]+ sec
+    Start 4: goodbye
+    Test #4: goodbye ..........................   Passed    [0-9.]+ sec
+    Start 4: goodbye
+    Test #4: goodbye ..........................   Passed    [0-9.]+ sec
+    Start 4: goodbye
+4/4 Test #4: goodbye ..........................   Passed    [0-9.]+ sec
++
+75% tests passed, 1 tests failed out of 4
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+[	 ]+2 - test1 \(Failed\)$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/test1.cmake

@@ -0,0 +1,13 @@
+# This is run by test test1 in repeat-until-fail-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1.  When the number is 2, then the
+# code sends out a cmake error causing the test to fail
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(${COUNT} EQUAL 2)
+  message(FATAL_ERROR "this test fails on the 2nd run")
+endif()