From dffc307c81220ffc243abc3b87ecfd694bd4cd35 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Jun 2015 09:53:37 -0400 Subject: [PATCH 1/4] Tests: Teach RunCMake infrastructure to optionally timeout Add a RunCMake_TEST_TIMEOUT option that tests can set to cause RunCMake to limit the time it waits for the child process to finish. --- Tests/RunCMake/RunCMake.cmake | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Tests/RunCMake/RunCMake.cmake b/Tests/RunCMake/RunCMake.cmake index 70c0d6c37..46bc4945b 100644 --- a/Tests/RunCMake/RunCMake.cmake +++ b/Tests/RunCMake/RunCMake.cmake @@ -66,6 +66,11 @@ function(run_cmake test) else() set(actual_stderr_var actual_stderr) endif() + if(DEFINED RunCMake_TEST_TIMEOUT) + set(maybe_timeout TIMEOUT ${RunCMake_TEST_TIMEOUT}) + else() + set(maybe_timeout "") + endif() if(RunCMake_TEST_COMMAND) execute_process( COMMAND ${RunCMake_TEST_COMMAND} @@ -73,6 +78,7 @@ function(run_cmake test) OUTPUT_VARIABLE actual_stdout ERROR_VARIABLE ${actual_stderr_var} RESULT_VARIABLE actual_result + ${maybe_timeout} ) else() execute_process( @@ -87,6 +93,7 @@ function(run_cmake test) OUTPUT_VARIABLE actual_stdout ERROR_VARIABLE ${actual_stderr_var} RESULT_VARIABLE actual_result + ${maybe_timeout} ) endif() set(msg "") From 8bf5a80b9668a31b85aaac00d1682228bbbbac5e Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Jun 2015 09:41:58 -0400 Subject: [PATCH 2/4] cmSystemTools: Add StringToULong helper Convert a string to an unsigned integer and reject any extra input. --- Source/cmSystemTools.cxx | 9 +++++++++ Source/cmSystemTools.h | 1 + 2 files changed, 10 insertions(+) diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx index e2adabe5f..7230a64f9 100644 --- a/Source/cmSystemTools.cxx +++ b/Source/cmSystemTools.cxx @@ -2955,3 +2955,12 @@ bool cmSystemTools::StringToLong(const char* str, long* value) *value = strtol(str, &endp, 10); return (*endp == '\0') && (endp != str) && (errno == 0); } + +//---------------------------------------------------------------------------- +bool cmSystemTools::StringToULong(const char* str, unsigned long* value) +{ + errno = 0; + char *endp; + *value = strtoul(str, &endp, 10); + return (*endp == '\0') && (endp != str) && (errno == 0); +} diff --git a/Source/cmSystemTools.h b/Source/cmSystemTools.h index 6feb6c5bf..8ebb4e325 100644 --- a/Source/cmSystemTools.h +++ b/Source/cmSystemTools.h @@ -469,6 +469,7 @@ public: /** Convert string to long. Expected that the whole string is an integer */ static bool StringToLong(const char* str, long* value); + static bool StringToULong(const char* str, unsigned long* value); #ifdef _WIN32 struct WindowsFileRetry From 07c550caa20d4b1d6ebc08269d744ff6a45c0a6d Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Mon, 1 Jun 2015 09:42:44 -0400 Subject: [PATCH 3/4] cmCTestMultiProcessHandler: Refactor RUN_SERIAL implementation The original implementation of the RUN_SERIAL test property worked by having such a test consume all available processors. Instead use an explicit flag to indicate that a serial test is running. This avoids artificially inflating the number of processors a test is expected to consume. --- Source/CTest/cmCTestMultiProcessHandler.cxx | 35 ++++++++++++++++++--- Source/CTest/cmCTestMultiProcessHandler.h | 1 + 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index bd090db97..28aec3d4c 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -44,6 +44,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler() this->RunningCount = 0; this->StopTimePassed = false; this->HasCycles = false; + this->SerialTestRunning = false; } cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() @@ -172,6 +173,11 @@ void cmCTestMultiProcessHandler::LockResources(int index) this->LockedResources.insert( this->Properties[index]->LockedResources.begin(), this->Properties[index]->LockedResources.end()); + + if (this->Properties[index]->RunSerial) + { + this->SerialTestRunning = true; + } } //--------------------------------------------------------- @@ -198,11 +204,9 @@ inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) { size_t processors = static_cast(this->Properties[test]->Processors); - //If this is set to run serially, it must run alone. - //Also, if processors setting is set higher than the -j + //If processors setting is set higher than the -j //setting, we default to using all of the process slots. - if(this->Properties[test]->RunSerial - || processors > this->ParallelLevel) + if (processors > this->ParallelLevel) { processors = this->ParallelLevel; } @@ -248,9 +252,27 @@ void cmCTestMultiProcessHandler::StartNextTests() return; } + // Don't start any new tests if one with the RUN_SERIAL property + // is already running. + if (this->SerialTestRunning) + { + return; + } + TestList copy = this->SortedTests; for(TestList::iterator test = copy.begin(); test != copy.end(); ++test) { + // Take a nap if we're currently performing a RUN_SERIAL test. + if (this->SerialTestRunning) + { + break; + } + // We can only start a RUN_SERIAL test if no other tests are also running. + if (this->Properties[*test]->RunSerial && this->RunningCount > 0) + { + continue; + } + size_t processors = GetProcessorsUsed(*test); if(processors <= numToStart && this->StartTest(*test)) @@ -319,6 +341,11 @@ bool cmCTestMultiProcessHandler::CheckOutput() this->WriteCheckpoint(test); this->UnlockResources(test); this->RunningCount -= GetProcessorsUsed(test); + if (this->Properties[test]->RunSerial) + { + this->SerialTestRunning = false; + } + delete p; } return true; diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 6440fbc19..243318f52 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -121,6 +121,7 @@ protected: cmCTest* CTest; bool HasCycles; bool Quiet; + bool SerialTestRunning; }; #endif From f62d301b9257542f5460902c400af3f947f10a66 Mon Sep 17 00:00:00 2001 From: Betsy McPhail Date: Tue, 9 Jun 2015 08:50:44 -0400 Subject: [PATCH 4/4] ctest: Optionally avoid starting tests that may exceed a given CPU load Add a TestLoad setting to CTest that can be set via a new --test-load command-line option, CTEST_TEST_LOAD variable, or TEST_LOAD option to the ctest_test command. Teach cmCTestMultiProcessHandler to measure the CPU load and avoid starting tests that may take more than the spare load currently available. The expression + <= must be true to start a new test. Co-Author: Zack Galbreath --- Help/command/ctest_memcheck.rst | 1 + Help/command/ctest_test.rst | 8 ++ Help/manual/cmake-variables.7.rst | 1 + Help/manual/ctest.1.rst | 14 ++ Help/release/dev/ctest-test-load-option.rst | 9 ++ Help/variable/CTEST_TEST_LOAD.rst | 7 + Modules/DartConfiguration.tcl.in | 4 + Source/CTest/cmCTestGenericHandler.cxx | 2 + Source/CTest/cmCTestGenericHandler.h | 3 + Source/CTest/cmCTestMultiProcessHandler.cxx | 126 ++++++++++++++++-- Source/CTest/cmCTestMultiProcessHandler.h | 3 + Source/CTest/cmCTestTestCommand.cxx | 33 +++++ Source/CTest/cmCTestTestCommand.h | 1 + Source/CTest/cmCTestTestHandler.cxx | 8 ++ Source/cmCTest.cxx | 35 +++++ Source/cmCTest.h | 5 + Source/ctest.cxx | 1 + .../CTestCommandLine/RunCMakeTest.cmake | 33 +++++ .../test-load-fail-stderr.txt | 1 + .../test-load-fail-stdout.txt | 2 + .../test-load-invalid-stderr.txt | 1 + .../test-load-invalid-stdout.txt | 7 + .../test-load-pass-stderr.txt | 1 + .../test-load-pass-stdout.txt | 7 + .../ctest_test/CTestTestLoadFail-result.txt | 1 + .../ctest_test/CTestTestLoadFail-stderr.txt | 1 + .../ctest_test/CTestTestLoadFail-stdout.txt | 2 + .../CTestTestLoadInvalid-stderr.txt | 1 + .../CTestTestLoadInvalid-stdout.txt | 7 + .../ctest_test/CTestTestLoadPass-stdout.txt | 7 + Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 41 ++++++ .../ctest_test/TestLoadFail-result.txt | 1 + .../ctest_test/TestLoadFail-stderr.txt | 1 + .../ctest_test/TestLoadFail-stdout.txt | 2 + .../ctest_test/TestLoadInvalid-stderr.txt | 1 + .../ctest_test/TestLoadInvalid-stdout.txt | 7 + .../ctest_test/TestLoadOrder-stderr.txt | 1 + .../ctest_test/TestLoadOrder-stdout.txt | 7 + .../ctest_test/TestLoadPass-stdout.txt | 7 + Tests/RunCMake/ctest_test/test.cmake.in | 1 + 40 files changed, 393 insertions(+), 8 deletions(-) create mode 100644 Help/release/dev/ctest-test-load-option.rst create mode 100644 Help/variable/CTEST_TEST_LOAD.rst create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadFail-result.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt diff --git a/Help/command/ctest_memcheck.rst b/Help/command/ctest_memcheck.rst index 2800511a4..29bdf7d0e 100644 --- a/Help/command/ctest_memcheck.rst +++ b/Help/command/ctest_memcheck.rst @@ -14,6 +14,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`. [EXCLUDE_LABEL ] [INCLUDE_LABEL ] [PARALLEL_LEVEL ] + [TEST_LOAD ] [SCHEDULE_RANDOM ] [STOP_TIME ] [RETURN_VALUE ] diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index af422b644..162db69a3 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -14,6 +14,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [EXCLUDE_LABEL ] [INCLUDE_LABEL ] [PARALLEL_LEVEL ] + [TEST_LOAD ] [SCHEDULE_RANDOM ] [STOP_TIME ] [RETURN_VALUE ] @@ -61,6 +62,13 @@ The options are: Specify a positive number representing the number of tests to be run in parallel. +``TEST_LOAD `` + While running tests in parallel, try not to start tests when they + may cause the CPU load to pass above a given threshold. If not + specified the :variable:`CTEST_TEST_LOAD` variable will be checked, + and then the ``--test-load`` command-line argument to :manual:`ctest(1)`. + See also the ``TestLoad`` setting in the :ref:`CTest Test Step`. + ``SCHEDULE_RANDOM `` Launch tests in a random order. This may be useful for detecting implicit test dependencies. diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index 0e6222f25..adbc40bd6 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -386,6 +386,7 @@ Variables for CTest /variable/CTEST_SVN_COMMAND /variable/CTEST_SVN_OPTIONS /variable/CTEST_SVN_UPDATE_OPTIONS + /variable/CTEST_TEST_LOAD /variable/CTEST_TEST_TIMEOUT /variable/CTEST_TRIGGER_SITE /variable/CTEST_UPDATE_COMMAND diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index c91321b10..50c856a04 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -66,6 +66,13 @@ Options number of jobs. This option can also be set by setting the environment variable ``CTEST_PARALLEL_LEVEL``. +``--test-load `` + While running tests in parallel (e.g. with ``-j``), try not to start + tests when they may cause the CPU load to pass above a given threshold. + + When ``ctest`` is run as a `Dashboard Client`_ this sets the + ``TestLoad`` option of the `CTest Test Step`_. + ``-Q,--quiet`` Make ctest quiet. @@ -776,6 +783,13 @@ Arguments to the command may specify some of the step settings. Configuration settings include: +``TestLoad`` + While running tests in parallel (e.g. with ``-j``), try not to start + tests when they may cause the CPU load to pass above a given threshold. + + * `CTest Script`_ variable: :variable:`CTEST_TEST_LOAD` + * :module:`CTest` module variable: ``CTEST_TEST_LOAD`` + ``TimeOut`` The default timeout for each test if not specified by the :prop_test:`TIMEOUT` test property. diff --git a/Help/release/dev/ctest-test-load-option.rst b/Help/release/dev/ctest-test-load-option.rst new file mode 100644 index 000000000..069f49ae2 --- /dev/null +++ b/Help/release/dev/ctest-test-load-option.rst @@ -0,0 +1,9 @@ +ctest-test-load-option +---------------------- + +* CTest learned to optionally measure the CPU load during parallel + testing and avoid starting tests that may cause the load to exceed + a given threshold. See the :manual:`ctest(1)` command ``--test-load`` + option, the ``TestLoad`` setting of the :ref:`CTest Test Step`, + the :variable:`CTEST_TEST_LOAD` variable, and the ``TEST_LOAD`` + option of the :command:`ctest_test` command. diff --git a/Help/variable/CTEST_TEST_LOAD.rst b/Help/variable/CTEST_TEST_LOAD.rst new file mode 100644 index 000000000..80823fe4e --- /dev/null +++ b/Help/variable/CTEST_TEST_LOAD.rst @@ -0,0 +1,7 @@ +CTEST_TEST_LOAD +--------------- + +Specify the ``TestLoad`` setting in the :ref:`CTest Test Step` +of a :manual:`ctest(1)` dashboard client script. This sets the +default value for the ``TEST_LOAD`` option of the :command:`ctest_test` +command. diff --git a/Modules/DartConfiguration.tcl.in b/Modules/DartConfiguration.tcl.in index 37a0a408e..918b407bf 100644 --- a/Modules/DartConfiguration.tcl.in +++ b/Modules/DartConfiguration.tcl.in @@ -95,6 +95,10 @@ SlurmRunCommand: @SLURM_SRUN_COMMAND@ # Currently set to 25 minutes TimeOut: @DART_TESTING_TIMEOUT@ +# During parallel testing CTest will not start a new test if doing +# so would cause the system load to exceed this value. +TestLoad: @CTEST_TEST_LOAD@ + UseLaunchers: @CTEST_USE_LAUNCHERS@ CurlOptions: @CTEST_CURL_OPTIONS@ # warning, if you add new options here that have to do with submit, diff --git a/Source/CTest/cmCTestGenericHandler.cxx b/Source/CTest/cmCTestGenericHandler.cxx index 81eb0a808..ad79ba2b1 100644 --- a/Source/CTest/cmCTestGenericHandler.cxx +++ b/Source/CTest/cmCTestGenericHandler.cxx @@ -23,6 +23,7 @@ cmCTestGenericHandler::cmCTestGenericHandler() this->SubmitIndex = 0; this->AppendXML = false; this->Quiet = false; + this->TestLoad = 0; } //---------------------------------------------------------------------- @@ -70,6 +71,7 @@ void cmCTestGenericHandler::SetPersistentOption(const std::string& op, void cmCTestGenericHandler::Initialize() { this->AppendXML = false; + this->TestLoad = 0; this->Options.clear(); t_StringToString::iterator it; for ( it = this->PersistentOptions.begin(); diff --git a/Source/CTest/cmCTestGenericHandler.h b/Source/CTest/cmCTestGenericHandler.h index 8567dd7cd..4b7ae7901 100644 --- a/Source/CTest/cmCTestGenericHandler.h +++ b/Source/CTest/cmCTestGenericHandler.h @@ -89,6 +89,8 @@ public: void SetAppendXML(bool b) { this->AppendXML = b; } void SetQuiet(bool b) { this->Quiet = b; } bool GetQuiet() { return this->Quiet; } + void SetTestLoad(unsigned long load) { this->TestLoad = load; } + unsigned long GetTestLoad() const { return this->TestLoad; } protected: bool StartResultingXML(cmCTest::Part part, @@ -97,6 +99,7 @@ protected: bool AppendXML; bool Quiet; + unsigned long TestLoad; cmSystemTools::OutputOption HandlerVerbose; cmCTest *CTest; t_StringToString Options; diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 28aec3d4c..483218658 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -13,12 +13,15 @@ #include "cmProcess.h" #include "cmStandardIncludes.h" #include "cmCTest.h" +#include "cmCTestScriptHandler.h" #include "cmSystemTools.h" #include #include #include #include +#include #include +#include class TestComparator { @@ -40,6 +43,7 @@ private: cmCTestMultiProcessHandler::cmCTestMultiProcessHandler() { this->ParallelLevel = 1; + this->TestLoad = 0; this->Completed = 0; this->RunningCount = 0; this->StopTimePassed = false; @@ -84,6 +88,11 @@ void cmCTestMultiProcessHandler::SetParallelLevel(size_t level) this->ParallelLevel = level < 1 ? 1 : level; } +void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load) +{ + this->TestLoad = load; +} + //--------------------------------------------------------- void cmCTestMultiProcessHandler::RunTests() { @@ -213,6 +222,11 @@ inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) return processors; } +std::string cmCTestMultiProcessHandler::GetName(int test) +{ + return this->Properties[test]->Name; +} + //--------------------------------------------------------- bool cmCTestMultiProcessHandler::StartTest(int test) { @@ -259,6 +273,46 @@ void cmCTestMultiProcessHandler::StartNextTests() return; } + bool allTestsFailedTestLoadCheck = false; + bool usedFakeLoadForTesting = false; + size_t minProcessorsRequired = this->ParallelLevel; + std::string testWithMinProcessors = ""; + + cmsys::SystemInformation info; + + unsigned long systemLoad = 0; + size_t spareLoad = 0; + if (this->TestLoad > 0) + { + // Activate possible wait. + allTestsFailedTestLoadCheck = true; + + // Check for a fake load average value used in testing. + if (const char* fake_load_value = + cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING")) + { + usedFakeLoadForTesting = true; + if (!cmSystemTools::StringToULong(fake_load_value, &systemLoad)) + { + cmSystemTools::Error("Failed to parse fake load value: ", + fake_load_value); + } + } + // If it's not set, look up the true load average. + else + { + systemLoad = static_cast(ceil(info.GetLoadAverage())); + } + spareLoad = (this->TestLoad > systemLoad ? + this->TestLoad - systemLoad : 0); + + // Don't start more tests than the spare load can support. + if (numToStart > spareLoad) + { + numToStart = spareLoad; + } + } + TestList copy = this->SortedTests; for(TestList::iterator test = copy.begin(); test != copy.end(); ++test) { @@ -274,18 +328,74 @@ void cmCTestMultiProcessHandler::StartNextTests() } size_t processors = GetProcessorsUsed(*test); - - if(processors <= numToStart && this->StartTest(*test)) + bool testLoadOk = true; + if (this->TestLoad > 0) { - if(this->StopTimePassed) - { - return; - } - numToStart -= processors; + if (processors <= spareLoad) + { + cmCTestLog(this->CTest, DEBUG, + "OK to run " << GetName(*test) << + ", it requires " << processors << + " procs & system load is: " << + systemLoad << std::endl); + allTestsFailedTestLoadCheck = false; + } + else + { + testLoadOk = false; + } + } + + if (processors <= minProcessorsRequired) + { + minProcessorsRequired = processors; + testWithMinProcessors = GetName(*test); + } + + if(testLoadOk && processors <= numToStart && this->StartTest(*test)) + { + if(this->StopTimePassed) + { + return; + } + + numToStart -= processors; } else if(numToStart == 0) { - return; + break; + } + } + + if (allTestsFailedTestLoadCheck) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, "); + if (this->SerialTestRunning) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, + "Waiting for RUN_SERIAL test to finish."); + } + else + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, + "System Load: " << systemLoad << ", " + "Max Allowed Load: " << this->TestLoad << ", " + "Smallest test " << testWithMinProcessors << + " requires " << minProcessorsRequired); + } + cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl); + + if (usedFakeLoadForTesting) + { + // Break out of the infinite loop of waiting for our fake load + // to come down. + this->StopTimePassed = true; + } + else + { + // Wait between 1 and 5 seconds before trying again. + cmCTestScriptHandler::SleepInSeconds( + cmSystemTools::RandomSeed() % 5 + 1); } } } diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 243318f52..ed3e155a0 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -37,6 +37,7 @@ public: void SetTests(TestMap& tests, PropertiesMap& properties); // Set the max number of tests that can be run at the same time. void SetParallelLevel(size_t); + void SetTestLoad(unsigned long load); virtual void RunTests(); void PrintTestList(); void PrintLabels(); @@ -93,6 +94,7 @@ protected: bool CheckCycles(); int FindMaxIndex(); inline size_t GetProcessorsUsed(int index); + std::string GetName(int index); void LockResources(int index); void UnlockResources(int index); @@ -116,6 +118,7 @@ protected: std::set LockedResources; std::vector* TestResults; size_t ParallelLevel; // max number of process that can be run at once + unsigned long TestLoad; std::set RunningTests; // current running tests cmCTestTestHandler * TestHandler; cmCTest* CTest; diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 8b357acd3..b7d8318f4 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -26,6 +26,7 @@ cmCTestTestCommand::cmCTestTestCommand() this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL"; this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM"; this->Arguments[ctt_STOP_TIME] = "STOP_TIME"; + this->Arguments[ctt_TEST_LOAD] = "TEST_LOAD"; this->Arguments[ctt_LAST] = 0; this->Last = ctt_LAST; } @@ -103,6 +104,38 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() { this->CTest->SetStopTime(this->Values[ctt_STOP_TIME]); } + + // Test load is determined by: TEST_LOAD argument, + // or CTEST_TEST_LOAD script variable, or ctest --test-load + // command line argument... in that order. + unsigned long testLoad; + const char* ctestTestLoad + = this->Makefile->GetDefinition("CTEST_TEST_LOAD"); + if(this->Values[ctt_TEST_LOAD] && *this->Values[ctt_TEST_LOAD]) + { + if (!cmSystemTools::StringToULong(this->Values[ctt_TEST_LOAD], &testLoad)) + { + testLoad = 0; + cmCTestLog(this->CTest, WARNING, "Invalid value for 'TEST_LOAD' : " + << this->Values[ctt_TEST_LOAD] << std::endl); + } + } + else if(ctestTestLoad && *ctestTestLoad) + { + if (!cmSystemTools::StringToULong(ctestTestLoad, &testLoad)) + { + testLoad = 0; + cmCTestLog(this->CTest, WARNING, + "Invalid value for 'CTEST_TEST_LOAD' : " << + ctestTestLoad << std::endl); + } + } + else + { + testLoad = this->CTest->GetTestLoad(); + } + handler->SetTestLoad(testLoad); + handler->SetQuiet(this->Quiet); return handler; } diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index a1e5f368f..0dfca974d 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -60,6 +60,7 @@ protected: ctt_PARALLEL_LEVEL, ctt_SCHEDULE_RANDOM, ctt_STOP_TIME, + ctt_TEST_LOAD, ctt_LAST }; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 70b7f5cb5..59576f4d9 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1062,6 +1062,14 @@ void cmCTestTestHandler::ProcessDirectory(std::vector &passed, parallel->SetParallelLevel(this->CTest->GetParallelLevel()); parallel->SetTestHandler(this); parallel->SetQuiet(this->Quiet); + if(this->TestLoad > 0) + { + parallel->SetTestLoad(this->TestLoad); + } + else + { + parallel->SetTestLoad(this->CTest->GetTestLoad()); + } *this->LogFile << "Start testing: " << this->CTest->CurrentTime() << std::endl diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index e3b7a2bcb..5887ba8ed 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -294,6 +294,7 @@ cmCTest::cmCTest() this->LabelSummary = true; this->ParallelLevel = 1; this->ParallelLevelSetInCli = false; + this->TestLoad = 0; this->SubmitIndex = 0; this->Failover = false; this->BatchJobs = false; @@ -393,6 +394,11 @@ void cmCTest::SetParallelLevel(int level) this->ParallelLevel = level < 1 ? 1 : level; } +void cmCTest::SetTestLoad(unsigned long load) +{ + this->TestLoad = load; +} + //---------------------------------------------------------------------------- bool cmCTest::ShouldCompressTestOutput() { @@ -820,6 +826,20 @@ bool cmCTest::UpdateCTestConfiguration() cmSystemTools::ChangeDirectory(this->BinaryDir); } this->TimeOut = atoi(this->GetCTestConfiguration("TimeOut").c_str()); + std::string const& testLoad = this->GetCTestConfiguration("TestLoad"); + if (!testLoad.empty()) + { + unsigned long load; + if (cmSystemTools::StringToULong(testLoad.c_str(), &load)) + { + this->SetTestLoad(load); + } + else + { + cmCTestLog(this, WARNING, "Invalid value for 'Test Load' : " + << testLoad << std::endl); + } + } if ( this->ProduceXML ) { this->CompressXMLFiles = cmSystemTools::IsOn( @@ -2051,6 +2071,21 @@ bool cmCTest::HandleCommandLineArguments(size_t &i, } } + if(this->CheckArgument(arg, "--test-load") && i < args.size() - 1) + { + i++; + unsigned long load; + if (cmSystemTools::StringToULong(args[i].c_str(), &load)) + { + this->SetTestLoad(load); + } + else + { + cmCTestLog(this, WARNING, + "Invalid value for 'Test Load' : " << args[i] << std::endl); + } + } + if(this->CheckArgument(arg, "--no-compress-output")) { this->CompressTestOutput = false; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index db3ea1063..47245ae00 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -161,6 +161,9 @@ public: int GetParallelLevel() { return this->ParallelLevel; } void SetParallelLevel(int); + unsigned long GetTestLoad() { return this->TestLoad; } + void SetTestLoad(unsigned long); + /** * Check if CTest file exists */ @@ -499,6 +502,8 @@ private: int ParallelLevel; bool ParallelLevelSetInCli; + unsigned long TestLoad; + int CompatibilityMode; // information for the --build-and-test options diff --git a/Source/ctest.cxx b/Source/ctest.cxx index e7847592d..afcbd613b 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -98,6 +98,7 @@ static const char * cmDocumentationOptions[][2] = {"--test-command", "The test to run with the --build-and-test option."}, {"--test-timeout", "The time limit in seconds, internal use only."}, + {"--test-load", "CPU load threshold for starting new parallel tests."}, {"--tomorrow-tag", "Nightly or experimental starts with next day tag."}, {"--ctest-config", "The configuration file used to initialize CTest state " "when submitting dashboards."}, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index a3ce1399c..aba1dafc2 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -1,4 +1,5 @@ include(RunCMake) +set(RunCMake_TEST_TIMEOUT 60) unset(ENV{CTEST_PARALLEL_LEVEL}) unset(ENV{CTEST_OUTPUT_ON_FAILURE}) @@ -52,3 +53,35 @@ add_test(MergeOutput \"${CMAKE_COMMAND}\" -P \"${RunCMake_SOURCE_DIR}/MergeOutpu run_cmake_command(MergeOutput ${CMAKE_CTEST_COMMAND} -V) endfunction() run_MergeOutput() + + +function(run_TestLoad name load) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " + add_test(TestLoad1 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\") + add_test(TestLoad2 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\") +") + run_cmake_command(${name} ${CMAKE_CTEST_COMMAND} -j2 --test-load ${load} --test-timeout 5) +endfunction() + +# Tests for the --test-load feature of ctest +# +# Spoof a load average value to make these tests more reliable. +set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +run_TestLoad(test-load-fail 2) + +# Verify that warning message is displayed but tests still start when +# an invalid argument is given. +run_TestLoad(test-load-invalid 'two') + +# Verify that new tests are started when the load average falls below +# our threshold. +run_TestLoad(test-load-pass 10) + +unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING}) diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt new file mode 100644 index 000000000..eafba1c69 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt new file mode 100644 index 000000000..153da09fb --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt @@ -0,0 +1,2 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test TestLoad[1-2] requires 1\*\*\*\*\* diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt new file mode 100644 index 000000000..caab3b9b7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt @@ -0,0 +1 @@ +Invalid value for 'Test Load' : 'two' diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt new file mode 100644 index 000000000..7ee3daee1 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt @@ -0,0 +1,7 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad + Start 1: TestLoad1 + Start 2: TestLoad2 +1/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec +2/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt new file mode 100644 index 000000000..10f32932e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt @@ -0,0 +1 @@ +^$ diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt new file mode 100644 index 000000000..7ee3daee1 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt @@ -0,0 +1,7 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad + Start 1: TestLoad1 + Start 2: TestLoad2 +1/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec +2/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt new file mode 100644 index 000000000..eafba1c69 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt new file mode 100644 index 000000000..e203c10c1 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt @@ -0,0 +1,2 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadFail-build +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 4, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$ diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt new file mode 100644 index 000000000..7f2d7f6df --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'CTEST_TEST_LOAD' : ERR2 diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt new file mode 100644 index 000000000..b54220cff --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt new file mode 100644 index 000000000..c221eed72 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadPass-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index d906290e0..21d0447e5 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -1,6 +1,8 @@ include(RunCTest) +set(RunCMake_TEST_TIMEOUT 60) set(CASE_CTEST_TEST_ARGS "") +set(CASE_CTEST_TEST_LOAD "") function(run_ctest_test CASE_NAME) set(CASE_CTEST_TEST_ARGS "${ARGN}") @@ -8,3 +10,42 @@ function(run_ctest_test CASE_NAME) endfunction() run_ctest_test(TestQuiet QUIET) + +# Tests for the 'Test Load' feature of ctest +# +# Spoof a load average value to make these tests more reliable. +set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5) + +# Verify that new tests are started when the load average falls below +# our threshold. +run_ctest_test(TestLoadPass TEST_LOAD 6) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +run_ctest_test(TestLoadFail TEST_LOAD 2) + +# Verify that when an invalid "TEST_LOAD" value is given, a warning +# message is displayed and the value is ignored. +run_ctest_test(TestLoadInvalid TEST_LOAD "ERR1") + +# Verify that new tests are started when the load average falls below +# our threshold. +set(CASE_CTEST_TEST_LOAD 7) +run_ctest_test(CTestTestLoadPass) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +set(CASE_CTEST_TEST_LOAD 4) +run_ctest_test(CTestTestLoadFail) + +# Verify that when an invalid "CTEST_TEST_LOAD" value is given, +# a warning message is displayed and the value is ignored. +set(CASE_CTEST_TEST_LOAD "ERR2") +run_ctest_test(CTestTestLoadInvalid) + +# Verify that the "TEST_LOAD" value has higher precedence than +# the "CTEST_TEST_LOAD" value +set(CASE_CTEST_TEST_LOAD "ERR3") +run_ctest_test(TestLoadOrder TEST_LOAD "ERR4") + +unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING}) diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-result.txt b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt new file mode 100644 index 000000000..eafba1c69 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt new file mode 100644 index 000000000..4d7ce48a4 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt @@ -0,0 +1,2 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadFail-build +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$ diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt new file mode 100644 index 000000000..40ddb3aec --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'TEST_LOAD' : ERR1 diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt new file mode 100644 index 000000000..c4fd35b4a --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadInvalid-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt new file mode 100644 index 000000000..1de730e75 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'TEST_LOAD' : ERR4 diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt new file mode 100644 index 000000000..22da09211 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadOrder-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt new file mode 100644 index 000000000..e5048f4b4 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadPass-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in index 1350abe00..a8de6a361 100644 --- a/Tests/RunCMake/ctest_test/test.cmake.in +++ b/Tests/RunCMake/ctest_test/test.cmake.in @@ -8,6 +8,7 @@ set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@") set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@") set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@") set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") +set(CTEST_TEST_LOAD "@CASE_CTEST_TEST_LOAD@") set(ctest_test_args "@CASE_CTEST_TEST_ARGS@") ctest_start(Experimental)