From fde70a1b26e130663cc4c02e0530d3aadc165a41 Mon Sep 17 00:00:00 2001 From: Bill Hoffman Date: Thu, 5 Mar 2015 16:51:10 -0500 Subject: [PATCH 1/2] ctest: Add a new --repeat-until-fail option This option tells ctest to run each test N times until the test fails or the N times have run. This is useful for finding random failing tests. --- Help/manual/ctest.1.rst | 5 ++ Source/CTest/cmCTestMultiProcessHandler.cxx | 13 +++- Source/CTest/cmCTestRunTest.cxx | 73 +++++++++++++++++-- Source/CTest/cmCTestRunTest.h | 9 +++ Source/cmCTest.cxx | 37 +++++++++- Source/cmCTest.h | 12 ++- Source/ctest.cxx | 2 + Tests/RunCMake/CMakeLists.txt | 1 + .../RunCMake/CTestCommandLine/CMakeLists.txt | 3 + .../CTestCommandLine/RunCMakeTest.cmake | 25 +++++++ Tests/RunCMake/CTestCommandLine/init.cmake | 3 + .../repeat-until-fail-bad1-result.txt | 1 + .../repeat-until-fail-bad1-stderr.txt | 1 + .../repeat-until-fail-bad2-result.txt | 1 + .../repeat-until-fail-bad2-stderr.txt | 1 + .../repeat-until-fail-cmake.cmake | 15 ++++ .../repeat-until-fail-ctest-result.txt | 1 + .../repeat-until-fail-ctest-stderr.txt | 1 + .../repeat-until-fail-ctest-stdout.txt | 30 ++++++++ .../repeat-until-fail-good-stderr.txt | 1 + Tests/RunCMake/CTestCommandLine/test1.cmake | 13 ++++ 21 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/CMakeLists.txt create mode 100644 Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/init.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-cmake.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test1.cmake diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index cc132c201..dd3bcfb70 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -194,6 +194,11 @@ Options subsequent calls to ctest with the --rerun-failed option will run the set of tests that most recently failed (if any). +``--repeat-until-fail `` + Require each test to run ```` times without failing in order to pass. + + This is useful in finding sporadic failures in test cases. + ``--max-width `` Set the max width for a test name to output diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index eb33d8edb..bd090db97 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test) this->RunningCount += GetProcessorsUsed(test); cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler); + if(this->CTest->GetRepeatUntilFail()) + { + testRun->SetRunUntilFailOn(); + testRun->SetNumberOfRuns(this->CTest->GetTestRepeat()); + } testRun->SetIndex(test); testRun->SetTestProperties(this->Properties[test]); @@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput() cmCTestRunTest* p = *i; int test = p->GetIndex(); - if(p->EndTest(this->Completed, this->Total, true)) + bool testResult = p->EndTest(this->Completed, this->Total, true); + if(p->StartAgain()) + { + this->Completed--; // remove the completed test because run again + continue; + } + if(testResult) { this->Passed->push_back(p->GetTestProperties()->Name); } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 03131fd74..6f72684d3 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler) this->CompressedOutput = ""; this->CompressionRatio = 2; this->StopTimePassed = false; + this->NumberOfRunsLeft = 1; // default to 1 run of the test + this->RunUntilFail = false; // default to run the test once + this->RunAgain = false; // default to not having to run again } cmCTestRunTest::~cmCTestRunTest() @@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) this->MemCheckPostProcess(); this->ComputeWeightedCost(); } - // Always push the current TestResult onto the + // If the test does not need to rerun push the current TestResult onto the // TestHandler vector - this->TestHandler->TestResults.push_back(this->TestResult); + if(!this->NeedsToRerun()) + { + this->TestHandler->TestResults.push_back(this->TestResult); + } delete this->TestProcess; return passed; } +bool cmCTestRunTest::StartAgain() +{ + if(!this->RunAgain) + { + return false; + } + this->RunAgain = false; // reset + // change to tests directory + std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory(); + cmSystemTools::ChangeDirectory(this->TestProperties->Directory); + this->StartTest(this->TotalNumberOfTests); + // change back + cmSystemTools::ChangeDirectory(current_dir); + return true; +} + +bool cmCTestRunTest::NeedsToRerun() +{ + this->NumberOfRunsLeft--; + if(this->NumberOfRunsLeft == 0) + { + return false; + } + // if number of runs left is not 0, and we are running until + // we find a failed test, then return true so the test can be + // restarted + if(this->RunUntilFail + && this->TestResult.Status == cmCTestTestHandler::COMPLETED) + { + this->RunAgain = true; + return true; + } + return false; +} //---------------------------------------------------------------------- void cmCTestRunTest::ComputeWeightedCost() { @@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess() // Starts the execution of a test. Returns once it has started bool cmCTestRunTest::StartTest(size_t total) { + this->TotalNumberOfTests = total; // save for rerun case cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8) << "Start " << std::setw(getNumWidth(this->TestHandler->GetMaxIndex())) @@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total) //---------------------------------------------------------------------- void cmCTestRunTest::ComputeArguments() { + this->Arguments.clear(); // reset becaue this might be a rerun std::vector::const_iterator j = this->TestProperties->Args.begin(); ++j; // skip test name - // find the test executable if(this->TestHandler->MemCheck) { @@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout, void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) { - cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) - << completed << "/"); - cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) - << total << " "); + // if this is the last or only run of this test + // then print out completed / total + // Only issue is if a test fails and we are running until fail + // then it will never print out the completed / total, same would + // got for run until pass. Trick is when this is called we don't + // yet know if we are passing or failing. + if(this->NumberOfRunsLeft == 1) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) + << completed << "/"); + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) + << total << " "); + } + // if this is one of several runs of a test just print blank space + // to keep things neat + else + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) + << " " << " "); + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total)) + << " " << " "); + } if ( this->TestHandler->MemCheck ) { diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 476f3e126..3b5c83160 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -27,6 +27,8 @@ public: cmCTestRunTest(cmCTestTestHandler* handler); ~cmCTestRunTest(); + void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;} + void SetRunUntilFailOn() { this->RunUntilFail = true;} void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop) { this->TestProperties = prop; } @@ -58,7 +60,10 @@ public: void ComputeArguments(); void ComputeWeightedCost(); + + bool StartAgain(); private: + bool NeedsToRerun(); void DartProcessing(); void ExeNotFound(std::string exe); // Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT) @@ -92,6 +97,10 @@ private: std::string ActualCommand; std::vector Arguments; bool StopTimePassed; + bool RunUntilFail; + int NumberOfRunsLeft; + bool RunAgain; + size_t TotalNumberOfTests; }; inline int getNumWidth(size_t n) diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index e6d396032..2cbdf9cbc 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -328,6 +328,8 @@ cmCTest::cmCTest() this->OutputTestOutputOnTestFailure = false; this->ComputedCompressTestOutput = false; this->ComputedCompressMemCheckOutput = false; + this->RepeatTests = 1; // default to run each test once + this->RepeatUntilFail = false; if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE")) { this->OutputTestOutputOnTestFailure = true; @@ -1983,11 +1985,11 @@ bool cmCTest::CheckArgument(const std::string& arg, const char* varg1, //---------------------------------------------------------------------- // Processes one command line argument (and its arguments if any) // for many simple options and then returns -void cmCTest::HandleCommandLineArguments(size_t &i, - std::vector &args) +bool cmCTest::HandleCommandLineArguments(size_t &i, + std::vector &args, + std::string& errormsg) { std::string arg = args[i]; - if(this->CheckArgument(arg, "-F")) { this->Failover = true; @@ -2005,6 +2007,27 @@ void cmCTest::HandleCommandLineArguments(size_t &i, this->SetParallelLevel(plevel); this->ParallelLevelSetInCli = true; } + if(this->CheckArgument(arg, "--repeat-until-fail")) + { + if( i >= args.size() - 1) + { + errormsg = "'--repeat-until-fail' requires an argument"; + return false; + } + i++; + long repeat = 1; + if(!cmSystemTools::StringToLong(args[i].c_str(), &repeat)) + { + errormsg = "'--repeat-until-fail' given non-integer value '" + + args[i] + "'"; + return false; + } + this->RepeatTests = static_cast(repeat); + if(repeat > 1) + { + this->RepeatUntilFail = true; + } + } if(this->CheckArgument(arg, "--no-compress-output")) { @@ -2190,6 +2213,7 @@ void cmCTest::HandleCommandLineArguments(size_t &i, this->GetHandler("test")->SetPersistentOption("RerunFailed", "true"); this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true"); } + return true; } //---------------------------------------------------------------------- @@ -2272,7 +2296,12 @@ int cmCTest::Run(std::vector &args, std::string* output) for(size_t i=1; i < args.size(); ++i) { // handle the simple commandline arguments - this->HandleCommandLineArguments(i,args); + std::string errormsg; + if(!this->HandleCommandLineArguments(i,args, errormsg)) + { + cmSystemTools::Error(errormsg.c_str()); + return 1; + } // handle the script arguments -S -SR -SP this->HandleScriptArguments(i,args,SRArgumentSpecified); diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 88191c4f3..3f033d9e3 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -429,8 +429,13 @@ public: { return this->Definitions; } - + // return the number of times a test should be run + int GetTestRepeat() { return this->RepeatTests;} + // return true if test should run until fail + bool GetRepeatUntilFail() { return this->RepeatUntilFail;} private: + int RepeatTests; + bool RepeatUntilFail; std::string ConfigType; std::string ScheduleType; std::string StopTime; @@ -535,8 +540,9 @@ private: bool AddVariableDefinition(const std::string &arg); //! parse and process most common command line arguments - void HandleCommandLineArguments(size_t &i, - std::vector &args); + bool HandleCommandLineArguments(size_t &i, + std::vector &args, + std::string& errormsg); //! hande the -S -SP and -SR arguments void HandleScriptArguments(size_t &i, diff --git a/Source/ctest.cxx b/Source/ctest.cxx index c0eb8acc6..0fc47b71e 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -75,6 +75,8 @@ static const char * cmDocumentationOptions[][2] = "Run a specific number of tests by number."}, {"-U, --union", "Take the Union of -I and -R"}, {"--rerun-failed", "Run only the tests that failed previously"}, + {"--repeat-until-fail ", "Require each test to run " + "times without failing in order to pass"}, {"--max-width ", "Set the max width for a test name to output"}, {"--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1."}, {"--no-label-summary", "Disable timing summary information for labels."}, diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index eb4205740..ffda31f20 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -198,6 +198,7 @@ add_RunCMake_test(CommandLine) add_RunCMake_test(install) add_RunCMake_test(CPackInstallProperties) add_RunCMake_test(ExternalProject) +add_RunCMake_test(CTestCommandLine) set(IfacePaths_INCLUDE_DIRECTORIES_ARGS -DTEST_PROP=INCLUDE_DIRECTORIES) add_RunCMake_test(IfacePaths_INCLUDE_DIRECTORIES TEST_DIR IfacePaths) diff --git a/Tests/RunCMake/CTestCommandLine/CMakeLists.txt b/Tests/RunCMake/CTestCommandLine/CMakeLists.txt new file mode 100644 index 000000000..289710955 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.0) +project(${RunCMake_TEST} NONE) +include(${RunCMake_TEST}.cmake) diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake new file mode 100644 index 000000000..2e5156cd6 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -0,0 +1,25 @@ +include(RunCMake) + +run_cmake_command(repeat-until-fail-bad1 + ${CMAKE_CTEST_COMMAND} --repeat-until-fail + ) +run_cmake_command(repeat-until-fail-bad2 + ${CMAKE_CTEST_COMMAND} --repeat-until-fail foo + ) +run_cmake_command(repeat-until-fail-good + ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 + ) + +function(run_repeat_until_fail_tests) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + + run_cmake(repeat-until-fail-cmake) + run_cmake_command(repeat-until-fail-ctest + ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3 + ) +endfunction() +run_repeat_until_fail_tests() diff --git a/Tests/RunCMake/CTestCommandLine/init.cmake b/Tests/RunCMake/CTestCommandLine/init.cmake new file mode 100644 index 000000000..a900f675c --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/init.cmake @@ -0,0 +1,3 @@ +# This is run by test initialization in repeat-until-fail-cmake.cmake +# with cmake -P. It creates TEST_OUTPUT_FILE with a 0 in it. +file(WRITE "${TEST_OUTPUT_FILE}" "0") diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt new file mode 100644 index 000000000..5ea8816e9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-fail' requires an argument$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt new file mode 100644 index 000000000..a79faaef5 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-fail' given non-integer value 'foo'$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-cmake.cmake new file mode 100644 index 000000000..465441645 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-cmake.cmake @@ -0,0 +1,15 @@ +enable_testing() + +set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") +add_test(NAME initialization + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake") +add_test(NAME test1 + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test1.cmake") +set_tests_properties(test1 PROPERTIES DEPENDS "initialization") + +add_test(hello ${CMAKE_COMMAND} -E echo hello) +add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-result.txt new file mode 100644 index 000000000..45a4fb75d --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-result.txt @@ -0,0 +1 @@ +8 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stderr.txt new file mode 100644 index 000000000..759378390 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stderr.txt @@ -0,0 +1 @@ +^Errors while running CTest$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stdout.txt new file mode 100644 index 000000000..0bc4f701b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-ctest-stdout.txt @@ -0,0 +1,30 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-fail-build + Start 1: initialization + Test #1: initialization ................... Passed [0-9.]+ sec + Start 1: initialization + Test #1: initialization ................... Passed [0-9.]+ sec + Start 1: initialization +1/4 Test #1: initialization ................... Passed [0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................ Passed [0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................\*\*\*Failed [0-9.]+ sec + Start 3: hello + Test #3: hello ............................ Passed [0-9.]+ sec + Start 3: hello + Test #3: hello ............................ Passed [0-9.]+ sec + Start 3: hello +3/4 Test #3: hello ............................ Passed [0-9.]+ sec + Start 4: goodbye + Test #4: goodbye .......................... Passed [0-9.]+ sec + Start 4: goodbye + Test #4: goodbye .......................... Passed [0-9.]+ sec + Start 4: goodbye +4/4 Test #4: goodbye .......................... Passed [0-9.]+ sec ++ +75% tests passed, 1 tests failed out of 4 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests FAILED: +[ ]+2 - test1 \(Failed\)$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt new file mode 100644 index 000000000..a7c4b11f7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/test1.cmake b/Tests/RunCMake/CTestCommandLine/test1.cmake new file mode 100644 index 000000000..eeae7a204 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test1.cmake @@ -0,0 +1,13 @@ +# This is run by test test1 in repeat-until-fail-cmake.cmake with cmake -P. +# It reads the file TEST_OUTPUT_FILE and increments the number +# found in the file by 1. When the number is 2, then the +# code sends out a cmake error causing the test to fail +# the second time it is run. +message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}") +file(READ "${TEST_OUTPUT_FILE}" COUNT) +message("COUNT= ${COUNT}") +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}") +if(${COUNT} EQUAL 2) + message(FATAL_ERROR "this test fails on the 2nd run") +endif() From 6bce027662588b972602e2e74c16a1883ed853cf Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 17 Mar 2015 11:39:43 -0400 Subject: [PATCH 2/2] Help: Add notes for topic 'ctest-repeat-until-fail' --- Help/release/dev/ctest-repeat-until-fail.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Help/release/dev/ctest-repeat-until-fail.rst diff --git a/Help/release/dev/ctest-repeat-until-fail.rst b/Help/release/dev/ctest-repeat-until-fail.rst new file mode 100644 index 000000000..8a679c61c --- /dev/null +++ b/Help/release/dev/ctest-repeat-until-fail.rst @@ -0,0 +1,5 @@ +ctest-repeat-until-fail +----------------------- + +* The :manual:`ctest(1)` tool learned a new ``--repeat-until-fail `` + option to help find sporadic test failures.