Checksums on CTest submit files, and retry timed out submissions.

Modules/CTest.cmake

@@ -163,6 +163,11 @@ IF(BUILD_TESTING)
   SET(DART_TESTING_TIMEOUT 1500 CACHE STRING 
     "Maximum time allowed before CTest will kill the test.")
 
+  SET(CTEST_SUBMIT_RETRY_DELAY 5 CACHE STRING
+    "How long to wait between timed-out CTest submissions.")
+  SET(CTEST_SUBMIT_RETRY_COUNT 3 CACHE STRING
+    "How many times to retry timed-out CTest submissions.")
+
   FIND_PROGRAM(MEMORYCHECK_COMMAND
     NAMES purify valgrind boundscheck
     PATHS
@@ -262,7 +267,9 @@ IF(BUILD_TESTING)
     SCPCOMMAND
     SLURM_SBATCH_COMMAND
     SLURM_SRUN_COMMAND
-    SITE 
+    SITE
+    CTEST_SUBMIT_RETRY_DELAY
+    CTEST_SUBMIT_RETRY_COUNT
     )
   #  BUILDNAME 
   IF(NOT RUN_FROM_DART)

Modules/DartConfiguration.tcl.in

@@ -83,3 +83,7 @@ CurlOptions: @CTEST_CURL_OPTIONS@
 # warning, if you add new options here that have to do with submit,
 # you have to update cmCTestSubmitCommand.cxx
 
+# For CTest submissions that timeout, these options
+# specify behavior for retrying the submission
+CTestRetryTime: @CTEST_SUBMIT_RETRY_DELAY@
+CTestRetryCount: @CTEST_SUBMIT_RETRY_COUNT@

Source/CTest/cmCTestSubmitCommand.cxx

@@ -147,6 +147,11 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
     static_cast<cmCTestSubmitHandler*>(handler)->SelectParts(this->Parts);
     }
 
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryTime",
+    this->RetryDelay.c_str());
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryCount",
+    this->RetryCount.c_str());
+
   return handler;
 }
 
@@ -169,6 +174,18 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
     return true;
     }
 
+  if(arg == "RETRY_COUNT")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryCount;
+    return true;
+    }
+
+  if(arg == "RETRY_DELAY")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryDelay;
+    return true;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentKeyword(arg);
 }
@@ -213,6 +230,16 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg)
     return true;
     }
 
+  if(this->ArgumentDoing == ArgumentDoingRetryCount)
+    {
+    this->RetryCount = arg;
+    }
+
+  if(this->ArgumentDoing == ArgumentDoingRetryDelay)
+    {
+    this->RetryDelay = arg;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentValue(arg);
 }

Source/CTest/cmCTestSubmitCommand.h

@@ -29,6 +29,8 @@ public:
     {
     this->PartsMentioned = false;
     this->FilesMentioned = false;
+    this->RetryCount = "";
+    this->RetryDelay = "";
     }
 
   /**
@@ -92,6 +94,8 @@ protected:
   {
     ArgumentDoingParts = Superclass::ArgumentDoingLast1,
     ArgumentDoingFiles,
+    ArgumentDoingRetryDelay,
+    ArgumentDoingRetryCount,
     ArgumentDoingLast2
   };
 
@@ -99,6 +103,8 @@ protected:
   std::set<cmCTest::Part> Parts;
   bool FilesMentioned;
   cmCTest::SetOfStrings Files;
+  std::string RetryCount;
+  std::string RetryDelay;
 };
 
 

Source/CTest/cmCTestSubmitHandler.cxx

@@ -15,6 +15,7 @@
 #include "cmVersion.h"
 #include "cmGeneratedFileStream.h"
 #include "cmCTest.h"
+#include "cmXMLParser.h"
 
 #include <cmsys/Process.h>
 #include <cmsys/Base64.h>
@@ -31,6 +32,80 @@
 
 typedef std::vector<char> cmCTestSubmitHandlerVectorOfChar;
 
+//----------------------------------------------------------------------------
+class cmCTestSubmitHandler::ResponseParser: public cmXMLParser
+{
+public:
+  ResponseParser() { this->Status = STATUS_OK; }
+  ~ResponseParser() {}
+
+public:
+
+  enum StatusType
+    {
+    STATUS_OK,
+    STATUS_WARNING,
+    STATUS_ERROR
+    };
+
+  StatusType Status;
+  std::string CDashVersion;
+  std::string Filename;
+  std::string MD5;
+  std::string Message;
+
+private:
+  std::string CurrentValue;
+  std::string CurrentTag;
+
+  virtual void StartElement(const char* name, const char** atts)
+    {
+    this->CurrentValue = "";
+    if(strcmp(name, "cdash") == 0)
+      {
+      this->CDashVersion = this->FindAttribute(atts, "version");
+      }
+    }
+
+  virtual void CharacterDataHandler(const char* data, int length)
+    {
+    this->CurrentValue.insert(this->CurrentValue.end(), data, data+length);
+    }
+
+  virtual void EndElement(const char* name)
+    {
+    if(strcmp(name, "status") == 0)
+      {
+      this->CurrentValue = cmSystemTools::UpperCase(this->CurrentValue);
+      if(this->CurrentValue == "OK" || this->CurrentValue == "SUCCESS")
+        {
+        this->Status = STATUS_OK;
+        }
+      else if(this->CurrentValue == "WARNING")
+        {
+        this->Status = STATUS_WARNING;
+        }
+      else
+        {
+        this->Status = STATUS_ERROR;
+        }
+      }
+    else if(strcmp(name, "filename") == 0)
+      {
+      this->Filename = this->CurrentValue;
+      }
+    else if(strcmp(name, "md5") == 0)
+      {
+      this->MD5 = this->CurrentValue;
+      }
+    else if(strcmp(name, "message") == 0)
+      {
+      this->Message = this->CurrentValue;
+      }
+    }
+};
+
+
 static size_t
 cmCTestSubmitHandlerWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
   void *data)
@@ -367,6 +442,13 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         = url + ((url.find("?",0) == cmStdString::npos) ? "?" : "&")
         + "FileName=" + ofile;
 
+      char md5[33];
+      cmSystemTools::ComputeFileMD5(local_file.c_str(), md5);
+      md5[32] = 0;
+      std::stringstream md5string;
+      md5string << "&MD5=" << md5;
+      upload_as += md5string.str();
+
       struct stat st;
       if ( ::stat(local_file.c_str(), &st) )
         {
@@ -382,7 +464,6 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         << local_file.c_str() << " to "
         << upload_as.c_str() << " Size: " << st.st_size << std::endl);
 
-
       // specify target
       ::curl_easy_setopt(curl,CURLOPT_URL, upload_as.c_str());
 
@@ -411,6 +492,47 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
       // Now run off and do what you've been told!
       res = ::curl_easy_perform(curl);
 
+      // If we time out or operation is too slow, wait and retry
+      if(res == CURLE_OPERATION_TIMEOUTED)
+        {
+        std::string retryTime = this->GetOption("RetryTime") == NULL ?
+          "" : this->GetOption("RetryTime");
+        std::string retryCount = this->GetOption("RetryCount") == NULL ?
+          "" : this->GetOption("RetryCount");
+
+        int time = retryTime == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestRetryTime").c_str()) : atoi(retryTime.c_str());
+        int count = retryCount == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestRetryCount").c_str()) : atoi(retryCount.c_str());
+
+        for(int i = 0; i < count; i++)
+          {
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Connection timed out, waiting " << time << " seconds...\n");
+
+          double stop = cmSystemTools::GetTime() + time;
+          while(cmSystemTools::GetTime() < stop) {} //wait <time> seconds
+
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Retry submission: Attempt " << (i + 1) << " of "
+            << count << std::endl);
+
+          ::fclose(ftpfile);
+          ftpfile = ::fopen(local_file.c_str(), "rb");
+          ::curl_easy_setopt(curl, CURLOPT_INFILE, ftpfile);
+
+          chunk.clear();
+          chunkDebug.clear();
+
+          res = ::curl_easy_perform(curl);
+
+          if(res != CURLE_OPERATION_TIMEDOUT)
+            {
+            break;
+            }
+          }
+        }
+
       if ( chunk.size() > 0 )
         {
         cmCTestLog(this->CTest, DEBUG, "CURL output: ["
@@ -467,29 +589,39 @@ void cmCTestSubmitHandler
 ::ParseResponse(cmCTestSubmitHandlerVectorOfChar chunk)
 {
   std::string output = "";
+  output.append(chunk.begin(), chunk.end());
 
-  for(cmCTestSubmitHandlerVectorOfChar::iterator i = chunk.begin();
-      i != chunk.end(); ++i)
+  if(output.find("<cdash") != output.npos)
     {
-    output += *i;
+    ResponseParser parser;
+    parser.Parse(output.c_str());
+
+    if(parser.Status != ResponseParser::STATUS_OK)
+      {
+      this->HasErrors = true;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Submission failed: " <<
+        parser.Message << std::endl);
+      return;
+      }
     }
-  output = cmSystemTools::UpperCase(output);
-  
-  if(output.find("WARNING") != std::string::npos)
+  else
     {
-    this->HasWarnings = true;
+    output = cmSystemTools::UpperCase(output);
+    if(output.find("WARNING") != std::string::npos)
+      {
+      this->HasWarnings = true;
+      }
+    if(output.find("ERROR") != std::string::npos)
+      {
+      this->HasErrors = true;
+      }
+
+    if(this->HasWarnings || this->HasErrors)
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Server Response:\n" <<
+            cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "\n");
+      }
     }
-  if(output.find("ERROR") != std::string::npos)
-    {
-    this->HasErrors = true;
-    }
-  
-  if(this->HasWarnings || this->HasErrors)
-    {
-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Server Response:\n" <<
-          cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "\n");
-    }
-  
 }
 
 //----------------------------------------------------------------------------

Source/CTest/cmCTestSubmitHandler.h

@@ -79,6 +79,7 @@ private:
 
   std::string GetSubmitResultsPrefix();
 
+  class         ResponseParser;
   cmStdString   HTTPProxy;
   int           HTTPProxyType;
   cmStdString   HTTPProxyAuth;

Tests/CTestTest3/test.cmake.in

@@ -113,7 +113,7 @@ IF(svncommand)
   CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
   CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
   CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
-  CTEST_SUBMIT(RETURN_VALUE res)
+  CTEST_SUBMIT(RETRY_COUNT 4 RETRY_DELAY 10 RETURN_VALUE res)
 
 ELSE(svncommand)
   MESSAGE("Cannot find SVN command: ${svncommand}")