diff --git a/Modules/CTest.cmake b/Modules/CTest.cmake index 2d0702e96..bdaf911b5 100644 --- a/Modules/CTest.cmake +++ b/Modules/CTest.cmake @@ -163,6 +163,11 @@ IF(BUILD_TESTING) SET(DART_TESTING_TIMEOUT 1500 CACHE STRING "Maximum time allowed before CTest will kill the test.") + SET(CTEST_SUBMIT_RETRY_DELAY 5 CACHE STRING + "How long to wait between timed-out CTest submissions.") + SET(CTEST_SUBMIT_RETRY_COUNT 3 CACHE STRING + "How many times to retry timed-out CTest submissions.") + FIND_PROGRAM(MEMORYCHECK_COMMAND NAMES purify valgrind boundscheck PATHS @@ -262,7 +267,9 @@ IF(BUILD_TESTING) SCPCOMMAND SLURM_SBATCH_COMMAND SLURM_SRUN_COMMAND - SITE + SITE + CTEST_SUBMIT_RETRY_DELAY + CTEST_SUBMIT_RETRY_COUNT ) # BUILDNAME IF(NOT RUN_FROM_DART) diff --git a/Modules/DartConfiguration.tcl.in b/Modules/DartConfiguration.tcl.in index 85b4138ba..799ff79a0 100644 --- a/Modules/DartConfiguration.tcl.in +++ b/Modules/DartConfiguration.tcl.in @@ -83,3 +83,7 @@ CurlOptions: @CTEST_CURL_OPTIONS@ # warning, if you add new options here that have to do with submit, # you have to update cmCTestSubmitCommand.cxx +# For CTest submissions that timeout, these options +# specify behavior for retrying the submission +CTestRetryTime: @CTEST_SUBMIT_RETRY_DELAY@ +CTestRetryCount: @CTEST_SUBMIT_RETRY_COUNT@ diff --git a/Source/CTest/cmCTestSubmitCommand.cxx b/Source/CTest/cmCTestSubmitCommand.cxx index d1226da1f..6a45d5882 100644 --- a/Source/CTest/cmCTestSubmitCommand.cxx +++ b/Source/CTest/cmCTestSubmitCommand.cxx @@ -147,6 +147,11 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() static_cast(handler)->SelectParts(this->Parts); } + static_cast(handler)->SetOption("RetryTime", + this->RetryDelay.c_str()); + static_cast(handler)->SetOption("RetryCount", + this->RetryCount.c_str()); + return handler; } @@ -169,6 +174,18 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg) return true; } + if(arg == "RETRY_COUNT") + { + this->ArgumentDoing = ArgumentDoingRetryCount; + return true; + } + + if(arg == "RETRY_DELAY") + { + this->ArgumentDoing = ArgumentDoingRetryDelay; + return true; + } + // Look for other arguments. return this->Superclass::CheckArgumentKeyword(arg); } @@ -213,6 +230,16 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg) return true; } + if(this->ArgumentDoing == ArgumentDoingRetryCount) + { + this->RetryCount = arg; + } + + if(this->ArgumentDoing == ArgumentDoingRetryDelay) + { + this->RetryDelay = arg; + } + // Look for other arguments. return this->Superclass::CheckArgumentValue(arg); } diff --git a/Source/CTest/cmCTestSubmitCommand.h b/Source/CTest/cmCTestSubmitCommand.h index ccaef7e36..14dfa37c0 100644 --- a/Source/CTest/cmCTestSubmitCommand.h +++ b/Source/CTest/cmCTestSubmitCommand.h @@ -29,6 +29,8 @@ public: { this->PartsMentioned = false; this->FilesMentioned = false; + this->RetryCount = ""; + this->RetryDelay = ""; } /** @@ -92,6 +94,8 @@ protected: { ArgumentDoingParts = Superclass::ArgumentDoingLast1, ArgumentDoingFiles, + ArgumentDoingRetryDelay, + ArgumentDoingRetryCount, ArgumentDoingLast2 }; @@ -99,6 +103,8 @@ protected: std::set Parts; bool FilesMentioned; cmCTest::SetOfStrings Files; + std::string RetryCount; + std::string RetryDelay; }; diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index 7b4f38b2c..fca05aca1 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -15,6 +15,7 @@ #include "cmVersion.h" #include "cmGeneratedFileStream.h" #include "cmCTest.h" +#include "cmXMLParser.h" #include #include @@ -31,6 +32,80 @@ typedef std::vector cmCTestSubmitHandlerVectorOfChar; +//---------------------------------------------------------------------------- +class cmCTestSubmitHandler::ResponseParser: public cmXMLParser +{ +public: + ResponseParser() { this->Status = STATUS_OK; } + ~ResponseParser() {} + +public: + + enum StatusType + { + STATUS_OK, + STATUS_WARNING, + STATUS_ERROR + }; + + StatusType Status; + std::string CDashVersion; + std::string Filename; + std::string MD5; + std::string Message; + +private: + std::string CurrentValue; + std::string CurrentTag; + + virtual void StartElement(const char* name, const char** atts) + { + this->CurrentValue = ""; + if(strcmp(name, "cdash") == 0) + { + this->CDashVersion = this->FindAttribute(atts, "version"); + } + } + + virtual void CharacterDataHandler(const char* data, int length) + { + this->CurrentValue.insert(this->CurrentValue.end(), data, data+length); + } + + virtual void EndElement(const char* name) + { + if(strcmp(name, "status") == 0) + { + this->CurrentValue = cmSystemTools::UpperCase(this->CurrentValue); + if(this->CurrentValue == "OK" || this->CurrentValue == "SUCCESS") + { + this->Status = STATUS_OK; + } + else if(this->CurrentValue == "WARNING") + { + this->Status = STATUS_WARNING; + } + else + { + this->Status = STATUS_ERROR; + } + } + else if(strcmp(name, "filename") == 0) + { + this->Filename = this->CurrentValue; + } + else if(strcmp(name, "md5") == 0) + { + this->MD5 = this->CurrentValue; + } + else if(strcmp(name, "message") == 0) + { + this->Message = this->CurrentValue; + } + } +}; + + static size_t cmCTestSubmitHandlerWriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) @@ -367,6 +442,13 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix, = url + ((url.find("?",0) == cmStdString::npos) ? "?" : "&") + "FileName=" + ofile; + char md5[33]; + cmSystemTools::ComputeFileMD5(local_file.c_str(), md5); + md5[32] = 0; + std::stringstream md5string; + md5string << "&MD5=" << md5; + upload_as += md5string.str(); + struct stat st; if ( ::stat(local_file.c_str(), &st) ) { @@ -382,7 +464,6 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix, << local_file.c_str() << " to " << upload_as.c_str() << " Size: " << st.st_size << std::endl); - // specify target ::curl_easy_setopt(curl,CURLOPT_URL, upload_as.c_str()); @@ -411,6 +492,47 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix, // Now run off and do what you've been told! res = ::curl_easy_perform(curl); + // If we time out or operation is too slow, wait and retry + if(res == CURLE_OPERATION_TIMEOUTED) + { + std::string retryTime = this->GetOption("RetryTime") == NULL ? + "" : this->GetOption("RetryTime"); + std::string retryCount = this->GetOption("RetryCount") == NULL ? + "" : this->GetOption("RetryCount"); + + int time = retryTime == "" ? atoi(this->CTest->GetCTestConfiguration( + "CTestRetryTime").c_str()) : atoi(retryTime.c_str()); + int count = retryCount == "" ? atoi(this->CTest->GetCTestConfiguration( + "CTestRetryCount").c_str()) : atoi(retryCount.c_str()); + + for(int i = 0; i < count; i++) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, + " Connection timed out, waiting " << time << " seconds...\n"); + + double stop = cmSystemTools::GetTime() + time; + while(cmSystemTools::GetTime() < stop) {} //wait