Checksums on CTest submit files, and retry timed out submissions.
This commit is contained in:
parent
7f619608d5
commit
e525649a4e
|
@ -163,6 +163,11 @@ IF(BUILD_TESTING)
|
|||
SET(DART_TESTING_TIMEOUT 1500 CACHE STRING
|
||||
"Maximum time allowed before CTest will kill the test.")
|
||||
|
||||
SET(CTEST_SUBMIT_RETRY_DELAY 5 CACHE STRING
|
||||
"How long to wait between timed-out CTest submissions.")
|
||||
SET(CTEST_SUBMIT_RETRY_COUNT 3 CACHE STRING
|
||||
"How many times to retry timed-out CTest submissions.")
|
||||
|
||||
FIND_PROGRAM(MEMORYCHECK_COMMAND
|
||||
NAMES purify valgrind boundscheck
|
||||
PATHS
|
||||
|
@ -263,6 +268,8 @@ IF(BUILD_TESTING)
|
|||
SLURM_SBATCH_COMMAND
|
||||
SLURM_SRUN_COMMAND
|
||||
SITE
|
||||
CTEST_SUBMIT_RETRY_DELAY
|
||||
CTEST_SUBMIT_RETRY_COUNT
|
||||
)
|
||||
# BUILDNAME
|
||||
IF(NOT RUN_FROM_DART)
|
||||
|
|
|
@ -83,3 +83,7 @@ CurlOptions: @CTEST_CURL_OPTIONS@
|
|||
# warning, if you add new options here that have to do with submit,
|
||||
# you have to update cmCTestSubmitCommand.cxx
|
||||
|
||||
# For CTest submissions that timeout, these options
|
||||
# specify behavior for retrying the submission
|
||||
CTestRetryTime: @CTEST_SUBMIT_RETRY_DELAY@
|
||||
CTestRetryCount: @CTEST_SUBMIT_RETRY_COUNT@
|
||||
|
|
|
@ -147,6 +147,11 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
|
|||
static_cast<cmCTestSubmitHandler*>(handler)->SelectParts(this->Parts);
|
||||
}
|
||||
|
||||
static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryTime",
|
||||
this->RetryDelay.c_str());
|
||||
static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryCount",
|
||||
this->RetryCount.c_str());
|
||||
|
||||
return handler;
|
||||
}
|
||||
|
||||
|
@ -169,6 +174,18 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
|
|||
return true;
|
||||
}
|
||||
|
||||
if(arg == "RETRY_COUNT")
|
||||
{
|
||||
this->ArgumentDoing = ArgumentDoingRetryCount;
|
||||
return true;
|
||||
}
|
||||
|
||||
if(arg == "RETRY_DELAY")
|
||||
{
|
||||
this->ArgumentDoing = ArgumentDoingRetryDelay;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Look for other arguments.
|
||||
return this->Superclass::CheckArgumentKeyword(arg);
|
||||
}
|
||||
|
@ -213,6 +230,16 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg)
|
|||
return true;
|
||||
}
|
||||
|
||||
if(this->ArgumentDoing == ArgumentDoingRetryCount)
|
||||
{
|
||||
this->RetryCount = arg;
|
||||
}
|
||||
|
||||
if(this->ArgumentDoing == ArgumentDoingRetryDelay)
|
||||
{
|
||||
this->RetryDelay = arg;
|
||||
}
|
||||
|
||||
// Look for other arguments.
|
||||
return this->Superclass::CheckArgumentValue(arg);
|
||||
}
|
||||
|
|
|
@ -29,6 +29,8 @@ public:
|
|||
{
|
||||
this->PartsMentioned = false;
|
||||
this->FilesMentioned = false;
|
||||
this->RetryCount = "";
|
||||
this->RetryDelay = "";
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -92,6 +94,8 @@ protected:
|
|||
{
|
||||
ArgumentDoingParts = Superclass::ArgumentDoingLast1,
|
||||
ArgumentDoingFiles,
|
||||
ArgumentDoingRetryDelay,
|
||||
ArgumentDoingRetryCount,
|
||||
ArgumentDoingLast2
|
||||
};
|
||||
|
||||
|
@ -99,6 +103,8 @@ protected:
|
|||
std::set<cmCTest::Part> Parts;
|
||||
bool FilesMentioned;
|
||||
cmCTest::SetOfStrings Files;
|
||||
std::string RetryCount;
|
||||
std::string RetryDelay;
|
||||
};
|
||||
|
||||
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
#include "cmVersion.h"
|
||||
#include "cmGeneratedFileStream.h"
|
||||
#include "cmCTest.h"
|
||||
#include "cmXMLParser.h"
|
||||
|
||||
#include <cmsys/Process.h>
|
||||
#include <cmsys/Base64.h>
|
||||
|
@ -31,6 +32,80 @@
|
|||
|
||||
typedef std::vector<char> cmCTestSubmitHandlerVectorOfChar;
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
class cmCTestSubmitHandler::ResponseParser: public cmXMLParser
|
||||
{
|
||||
public:
|
||||
ResponseParser() { this->Status = STATUS_OK; }
|
||||
~ResponseParser() {}
|
||||
|
||||
public:
|
||||
|
||||
enum StatusType
|
||||
{
|
||||
STATUS_OK,
|
||||
STATUS_WARNING,
|
||||
STATUS_ERROR
|
||||
};
|
||||
|
||||
StatusType Status;
|
||||
std::string CDashVersion;
|
||||
std::string Filename;
|
||||
std::string MD5;
|
||||
std::string Message;
|
||||
|
||||
private:
|
||||
std::string CurrentValue;
|
||||
std::string CurrentTag;
|
||||
|
||||
virtual void StartElement(const char* name, const char** atts)
|
||||
{
|
||||
this->CurrentValue = "";
|
||||
if(strcmp(name, "cdash") == 0)
|
||||
{
|
||||
this->CDashVersion = this->FindAttribute(atts, "version");
|
||||
}
|
||||
}
|
||||
|
||||
virtual void CharacterDataHandler(const char* data, int length)
|
||||
{
|
||||
this->CurrentValue.insert(this->CurrentValue.end(), data, data+length);
|
||||
}
|
||||
|
||||
virtual void EndElement(const char* name)
|
||||
{
|
||||
if(strcmp(name, "status") == 0)
|
||||
{
|
||||
this->CurrentValue = cmSystemTools::UpperCase(this->CurrentValue);
|
||||
if(this->CurrentValue == "OK" || this->CurrentValue == "SUCCESS")
|
||||
{
|
||||
this->Status = STATUS_OK;
|
||||
}
|
||||
else if(this->CurrentValue == "WARNING")
|
||||
{
|
||||
this->Status = STATUS_WARNING;
|
||||
}
|
||||
else
|
||||
{
|
||||
this->Status = STATUS_ERROR;
|
||||
}
|
||||
}
|
||||
else if(strcmp(name, "filename") == 0)
|
||||
{
|
||||
this->Filename = this->CurrentValue;
|
||||
}
|
||||
else if(strcmp(name, "md5") == 0)
|
||||
{
|
||||
this->MD5 = this->CurrentValue;
|
||||
}
|
||||
else if(strcmp(name, "message") == 0)
|
||||
{
|
||||
this->Message = this->CurrentValue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
static size_t
|
||||
cmCTestSubmitHandlerWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
|
||||
void *data)
|
||||
|
@ -367,6 +442,13 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
|
|||
= url + ((url.find("?",0) == cmStdString::npos) ? "?" : "&")
|
||||
+ "FileName=" + ofile;
|
||||
|
||||
char md5[33];
|
||||
cmSystemTools::ComputeFileMD5(local_file.c_str(), md5);
|
||||
md5[32] = 0;
|
||||
std::stringstream md5string;
|
||||
md5string << "&MD5=" << md5;
|
||||
upload_as += md5string.str();
|
||||
|
||||
struct stat st;
|
||||
if ( ::stat(local_file.c_str(), &st) )
|
||||
{
|
||||
|
@ -382,7 +464,6 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
|
|||
<< local_file.c_str() << " to "
|
||||
<< upload_as.c_str() << " Size: " << st.st_size << std::endl);
|
||||
|
||||
|
||||
// specify target
|
||||
::curl_easy_setopt(curl,CURLOPT_URL, upload_as.c_str());
|
||||
|
||||
|
@ -411,6 +492,47 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
|
|||
// Now run off and do what you've been told!
|
||||
res = ::curl_easy_perform(curl);
|
||||
|
||||
// If we time out or operation is too slow, wait and retry
|
||||
if(res == CURLE_OPERATION_TIMEOUTED)
|
||||
{
|
||||
std::string retryTime = this->GetOption("RetryTime") == NULL ?
|
||||
"" : this->GetOption("RetryTime");
|
||||
std::string retryCount = this->GetOption("RetryCount") == NULL ?
|
||||
"" : this->GetOption("RetryCount");
|
||||
|
||||
int time = retryTime == "" ? atoi(this->CTest->GetCTestConfiguration(
|
||||
"CTestRetryTime").c_str()) : atoi(retryTime.c_str());
|
||||
int count = retryCount == "" ? atoi(this->CTest->GetCTestConfiguration(
|
||||
"CTestRetryCount").c_str()) : atoi(retryCount.c_str());
|
||||
|
||||
for(int i = 0; i < count; i++)
|
||||
{
|
||||
cmCTestLog(this->CTest, HANDLER_OUTPUT,
|
||||
" Connection timed out, waiting " << time << " seconds...\n");
|
||||
|
||||
double stop = cmSystemTools::GetTime() + time;
|
||||
while(cmSystemTools::GetTime() < stop) {} //wait <time> seconds
|
||||
|
||||
cmCTestLog(this->CTest, HANDLER_OUTPUT,
|
||||
" Retry submission: Attempt " << (i + 1) << " of "
|
||||
<< count << std::endl);
|
||||
|
||||
::fclose(ftpfile);
|
||||
ftpfile = ::fopen(local_file.c_str(), "rb");
|
||||
::curl_easy_setopt(curl, CURLOPT_INFILE, ftpfile);
|
||||
|
||||
chunk.clear();
|
||||
chunkDebug.clear();
|
||||
|
||||
res = ::curl_easy_perform(curl);
|
||||
|
||||
if(res != CURLE_OPERATION_TIMEDOUT)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( chunk.size() > 0 )
|
||||
{
|
||||
cmCTestLog(this->CTest, DEBUG, "CURL output: ["
|
||||
|
@ -467,14 +589,24 @@ void cmCTestSubmitHandler
|
|||
::ParseResponse(cmCTestSubmitHandlerVectorOfChar chunk)
|
||||
{
|
||||
std::string output = "";
|
||||
output.append(chunk.begin(), chunk.end());
|
||||
|
||||
for(cmCTestSubmitHandlerVectorOfChar::iterator i = chunk.begin();
|
||||
i != chunk.end(); ++i)
|
||||
if(output.find("<cdash") != output.npos)
|
||||
{
|
||||
output += *i;
|
||||
}
|
||||
output = cmSystemTools::UpperCase(output);
|
||||
ResponseParser parser;
|
||||
parser.Parse(output.c_str());
|
||||
|
||||
if(parser.Status != ResponseParser::STATUS_OK)
|
||||
{
|
||||
this->HasErrors = true;
|
||||
cmCTestLog(this->CTest, HANDLER_OUTPUT, " Submission failed: " <<
|
||||
parser.Message << std::endl);
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
output = cmSystemTools::UpperCase(output);
|
||||
if(output.find("WARNING") != std::string::npos)
|
||||
{
|
||||
this->HasWarnings = true;
|
||||
|
@ -489,7 +621,7 @@ void cmCTestSubmitHandler
|
|||
cmCTestLog(this->CTest, HANDLER_OUTPUT, " Server Response:\n" <<
|
||||
cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "\n");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
|
|
|
@ -79,6 +79,7 @@ private:
|
|||
|
||||
std::string GetSubmitResultsPrefix();
|
||||
|
||||
class ResponseParser;
|
||||
cmStdString HTTPProxy;
|
||||
int HTTPProxyType;
|
||||
cmStdString HTTPProxyAuth;
|
||||
|
|
|
@ -113,7 +113,7 @@ IF(svncommand)
|
|||
CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
|
||||
CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
|
||||
CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
|
||||
CTEST_SUBMIT(RETURN_VALUE res)
|
||||
CTEST_SUBMIT(RETRY_COUNT 4 RETRY_DELAY 10 RETURN_VALUE res)
|
||||
|
||||
ELSE(svncommand)
|
||||
MESSAGE("Cannot find SVN command: ${svncommand}")
|
||||
|
|
Loading…
Reference in New Issue