Merge topic 'resolve/ctest-file-checksum/remove-CTestTest3'

38c762c Merge 'remove-CTestTest3' into ctest-file-checksum
46df0b4 Activate retry code on any curl submit failure.
8705497 Checksum test should use CMAKE_TESTS_CDASH_SERVER
d0d1cdd Mock checksum failure output for old CDash versions
af5ef0c Testing for CTest checksum
86e81b5 CTest should resubmit in the checksum failed case
d6b7107 Fix subscript out of range crash
082c87e Cross-platform fixes for checksum/retry code
e525649 Checksums on CTest submit files, and retry timed out submissions.

Modules/CTest.cmake

@@ -163,6 +163,11 @@ IF(BUILD_TESTING)
   SET(DART_TESTING_TIMEOUT 1500 CACHE STRING 
     "Maximum time allowed before CTest will kill the test.")
 
+  SET(CTEST_SUBMIT_RETRY_DELAY 5 CACHE STRING
+    "How long to wait between timed-out CTest submissions.")
+  SET(CTEST_SUBMIT_RETRY_COUNT 3 CACHE STRING
+    "How many times to retry timed-out CTest submissions.")
+
   FIND_PROGRAM(MEMORYCHECK_COMMAND
     NAMES purify valgrind boundscheck
     PATHS
@@ -262,7 +267,9 @@ IF(BUILD_TESTING)
     SCPCOMMAND
     SLURM_SBATCH_COMMAND
     SLURM_SRUN_COMMAND
-    SITE 
+    SITE
+    CTEST_SUBMIT_RETRY_DELAY
+    CTEST_SUBMIT_RETRY_COUNT
     )
   #  BUILDNAME 
   IF(NOT RUN_FROM_DART)

Modules/DartConfiguration.tcl.in

@@ -84,3 +84,7 @@ CurlOptions: @CTEST_CURL_OPTIONS@
 # warning, if you add new options here that have to do with submit,
 # you have to update cmCTestSubmitCommand.cxx
 
+# For CTest submissions that timeout, these options
+# specify behavior for retrying the submission
+CTestSubmitRetryDelay: @CTEST_SUBMIT_RETRY_DELAY@
+CTestSubmitRetryCount: @CTEST_SUBMIT_RETRY_COUNT@

Source/CTest/cmCTestSubmitCommand.cxx

@@ -147,6 +147,13 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
     static_cast<cmCTestSubmitHandler*>(handler)->SelectParts(this->Parts);
     }
 
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryDelay",
+    this->RetryDelay.c_str());
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryCount",
+    this->RetryCount.c_str());
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("InternalTest",
+    this->InternalTest ? "ON" : "OFF");
+
   return handler;
 }
 
@@ -169,6 +176,24 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
     return true;
     }
 
+  if(arg == "RETRY_COUNT")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryCount;
+    return true;
+    }
+
+  if(arg == "RETRY_DELAY")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryDelay;
+    return true;
+    }
+
+  if(arg == "INTERNAL_TEST_CHECKSUM")
+    {
+    this->InternalTest = true;
+    return true;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentKeyword(arg);
 }
@@ -213,6 +238,18 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg)
     return true;
     }
 
+  if(this->ArgumentDoing == ArgumentDoingRetryCount)
+    {
+    this->RetryCount = arg;
+    return true;
+    }
+
+  if(this->ArgumentDoing == ArgumentDoingRetryDelay)
+    {
+    this->RetryDelay = arg;
+    return true;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentValue(arg);
 }

Source/CTest/cmCTestSubmitCommand.h

@@ -29,6 +29,9 @@ public:
     {
     this->PartsMentioned = false;
     this->FilesMentioned = false;
+    this->InternalTest = false;
+    this->RetryCount = "";
+    this->RetryDelay = "";
     }
 
   /**
@@ -61,7 +64,8 @@ public:
   virtual const char* GetFullDocumentation()
     {
     return
-      "  ctest_submit([PARTS ...] [FILES ...] [RETURN_VALUE res])\n"
+      "  ctest_submit([PARTS ...] [FILES ...] [RETRY_COUNT count] "
+      "               [RETRY_DELAY delay][RETURN_VALUE res])\n"
       "By default all available parts are submitted if no PARTS or FILES "
       "are specified.  "
       "The PARTS option lists a subset of parts to be submitted.  "
@@ -77,7 +81,11 @@ public:
       "  ExtraFiles = Files listed by CTEST_EXTRA_SUBMIT_FILES\n"
       "  Submit     = nothing\n"
       "The FILES option explicitly lists specific files to be submitted.  "
-      "Each individual file must exist at the time of the call.\n";
+      "Each individual file must exist at the time of the call.\n"
+      "The RETRY_DELAY option specifies how long in seconds to wait after "
+      "a timed-out submission before attempting to re-submit.\n"
+      "The RETRY_COUNT option specifies how many times to retry a timed-out "
+      "submission.\n";
     }
 
   cmTypeMacro(cmCTestSubmitCommand, cmCTestHandlerCommand);
@@ -92,13 +100,18 @@ protected:
   {
     ArgumentDoingParts = Superclass::ArgumentDoingLast1,
     ArgumentDoingFiles,
+    ArgumentDoingRetryDelay,
+    ArgumentDoingRetryCount,
     ArgumentDoingLast2
   };
 
   bool PartsMentioned;
   std::set<cmCTest::Part> Parts;
   bool FilesMentioned;
+  bool InternalTest;
   cmCTest::SetOfStrings Files;
+  std::string RetryCount;
+  std::string RetryDelay;
 };
 
 

Source/CTest/cmCTestSubmitHandler.cxx

@@ -15,6 +15,7 @@
 #include "cmVersion.h"
 #include "cmGeneratedFileStream.h"
 #include "cmCTest.h"
+#include "cmXMLParser.h"
 
 #include <cmsys/Process.h>
 #include <cmsys/Base64.h>
@@ -31,6 +32,90 @@
 
 typedef std::vector<char> cmCTestSubmitHandlerVectorOfChar;
 
+//----------------------------------------------------------------------------
+class cmCTestSubmitHandler::ResponseParser: public cmXMLParser
+{
+public:
+  ResponseParser() { this->Status = STATUS_OK; }
+  ~ResponseParser() {}
+
+public:
+
+  enum StatusType
+    {
+    STATUS_OK,
+    STATUS_WARNING,
+    STATUS_ERROR
+    };
+
+  StatusType Status;
+  std::string CDashVersion;
+  std::string Filename;
+  std::string MD5;
+  std::string Message;
+
+private:
+
+  std::vector<char> CurrentValue;
+
+  std::string GetCurrentValue()
+    {
+    std::string val;
+    if(this->CurrentValue.size())
+      {
+      val.assign(&this->CurrentValue[0], this->CurrentValue.size());
+      }
+    return val;
+    }
+
+  virtual void StartElement(const char* name, const char** atts)
+    {
+    this->CurrentValue.clear();
+    if(strcmp(name, "cdash") == 0)
+      {
+      this->CDashVersion = this->FindAttribute(atts, "version");
+      }
+    }
+
+  virtual void CharacterDataHandler(const char* data, int length)
+    {
+    this->CurrentValue.insert(this->CurrentValue.end(), data, data+length);
+    }
+
+  virtual void EndElement(const char* name)
+    {
+    if(strcmp(name, "status") == 0)
+      {
+      std::string status = cmSystemTools::UpperCase(this->GetCurrentValue());
+      if(status == "OK" || status == "SUCCESS")
+        {
+        this->Status = STATUS_OK;
+        }
+      else if(status == "WARNING")
+        {
+        this->Status = STATUS_WARNING;
+        }
+      else
+        {
+        this->Status = STATUS_ERROR;
+        }
+      }
+    else if(strcmp(name, "filename") == 0)
+      {
+      this->Filename = this->GetCurrentValue();
+      }
+    else if(strcmp(name, "md5") == 0)
+      {
+      this->MD5 = this->GetCurrentValue();
+      }
+    else if(strcmp(name, "message") == 0)
+      {
+      this->Message = this->GetCurrentValue();
+      }
+    }
+};
+
+
 static size_t
 cmCTestSubmitHandlerWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
   void *data)
@@ -367,6 +452,20 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         = url + ((url.find("?",0) == cmStdString::npos) ? "?" : "&")
         + "FileName=" + ofile;
 
+      upload_as += "&MD5=";
+
+      if(cmSystemTools::IsOn(this->GetOption("InternalTest")))
+        {
+        upload_as += "bad_md5sum";
+        }
+      else
+        {
+        char md5[33];
+        cmSystemTools::ComputeFileMD5(local_file.c_str(), md5);
+        md5[32] = 0;
+        upload_as += md5;
+        }
+
       struct stat st;
       if ( ::stat(local_file.c_str(), &st) )
         {
@@ -382,7 +481,6 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         << local_file.c_str() << " to "
         << upload_as.c_str() << " Size: " << st.st_size << std::endl);
 
-
       // specify target
       ::curl_easy_setopt(curl,CURLOPT_URL, upload_as.c_str());
 
@@ -411,6 +509,19 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
       // Now run off and do what you've been told!
       res = ::curl_easy_perform(curl);
 
+      if(cmSystemTools::IsOn(this->GetOption("InternalTest")) &&
+         cmSystemTools::VersionCompare(cmSystemTools::OP_LESS,
+         this->CTest->GetCDashVersion().c_str(), "1.7"))
+        {
+        // mock failure output for internal test case
+        std::string mock_output = "<cdash version=\"1.7.0\">\n"
+          "  <status>ERROR</status>\n"
+          "  <message>Checksum failed for file.</message>\n"
+          "</cdash>\n";
+        chunk.clear();
+        chunk.assign(mock_output.begin(), mock_output.end());
+        }
+
       if ( chunk.size() > 0 )
         {
         cmCTestLog(this->CTest, DEBUG, "CURL output: ["
@@ -425,6 +536,60 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
           << std::endl);
         }
 
+      // If curl failed for any reason, or checksum fails, wait and retry
+      //
+      if(res != CURLE_OK || this->HasErrors)
+        {
+        std::string retryDelay = this->GetOption("RetryDelay") == NULL ?
+          "" : this->GetOption("RetryDelay");
+        std::string retryCount = this->GetOption("RetryCount") == NULL ?
+          "" : this->GetOption("RetryCount");
+
+        int delay = retryDelay == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestSubmitRetryDelay").c_str()) : atoi(retryDelay.c_str());
+        int count = retryCount == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestSubmitRetryCount").c_str()) : atoi(retryCount.c_str());
+
+        for(int i = 0; i < count; i++)
+          {
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Submit failed, waiting " << delay << " seconds...\n");
+
+          double stop = cmSystemTools::GetTime() + delay;
+          while(cmSystemTools::GetTime() < stop)
+            {
+            cmSystemTools::Delay(100);
+            }
+
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Retry submission: Attempt " << (i + 1) << " of "
+            << count << std::endl);
+
+          ::fclose(ftpfile);
+          ftpfile = ::fopen(local_file.c_str(), "rb");
+          ::curl_easy_setopt(curl, CURLOPT_INFILE, ftpfile);
+
+          chunk.clear();
+          chunkDebug.clear();
+          this->HasErrors = false;
+
+          res = ::curl_easy_perform(curl);
+
+          if ( chunk.size() > 0 )
+            {
+            cmCTestLog(this->CTest, DEBUG, "CURL output: ["
+              << cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "]"
+              << std::endl);
+            this->ParseResponse(chunk);
+            }
+
+          if(res == CURLE_OK && !this->HasErrors)
+            {
+            break;
+            }
+          }
+        }
+
       fclose(ftpfile);
       if ( res )
         {
@@ -467,14 +632,22 @@ void cmCTestSubmitHandler
 ::ParseResponse(cmCTestSubmitHandlerVectorOfChar chunk)
 {
   std::string output = "";
+  output.append(chunk.begin(), chunk.end());
 
-  for(cmCTestSubmitHandlerVectorOfChar::iterator i = chunk.begin();
+  if(output.find("<cdash") != output.npos)
-      i != chunk.end(); ++i)
     {
-    output += *i;
+    ResponseParser parser;
+    parser.Parse(output.c_str());
+
+    if(parser.Status != ResponseParser::STATUS_OK)
+      {
+      this->HasErrors = true;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Submission failed: " <<
+        parser.Message << std::endl);
+      return;
+      }
     }
   output = cmSystemTools::UpperCase(output);
-  
   if(output.find("WARNING") != std::string::npos)
     {
     this->HasWarnings = true;
@@ -483,13 +656,12 @@ void cmCTestSubmitHandler
     {
     this->HasErrors = true;
     }
-  
+
   if(this->HasWarnings || this->HasErrors)
     {
     cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Server Response:\n" <<
           cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "\n");
     }
-  
 }
 
 //----------------------------------------------------------------------------

Source/CTest/cmCTestSubmitHandler.h

@@ -79,6 +79,7 @@ private:
 
   std::string GetSubmitResultsPrefix();
 
+  class         ResponseParser;
   cmStdString   HTTPProxy;
   int           HTTPProxyType;
   cmStdString   HTTPProxyAuth;

Tests/CMakeLists.txt

@@ -1493,6 +1493,17 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
       --output-log "${CMake_BINARY_DIR}/Tests/CTestTest2/testOutput.log"
       )
 
+    CONFIGURE_FILE("${CMake_SOURCE_DIR}/Tests/CTestTestChecksum/test.cmake.in"
+      "${CMake_BINARY_DIR}/Tests/CTestTestChecksum/test.cmake" @ONLY
+      ESCAPE_QUOTES)
+    ADD_TEST(CTestTestChecksum ${CMAKE_CTEST_COMMAND}
+      -S "${CMake_BINARY_DIR}/Tests/CTestTestChecksum/test.cmake" -V
+      --output-log
+      "${CMake_BINARY_DIR}/Tests/CTestTestChecksum/testOutput.log"
+      )
+    SET_TESTS_PROPERTIES(CTestTestChecksum PROPERTIES PASS_REGULAR_EXPRESSION
+      "Submission failed: Checksum failed for file")
+
     # these tests take a long time, make sure they have it
     # if timeouts have not already been set
     GET_TEST_PROPERTY(CTestTest TIMEOUT PREVIOUS_TIMEOUT)

Tests/CTestTestChecksum/test.cmake.in

@@ -0,0 +1,28 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.1)
+
+# Settings:
+SET(CTEST_DASHBOARD_ROOT                "@CMake_BINARY_DIR@/Tests/CTestTest")
+SET(CTEST_SITE                          "@SITE@")
+SET(CTEST_BUILD_NAME                    "CTestTest-@BUILDNAME@-Checksum")
+
+SET(CTEST_SOURCE_DIRECTORY              "@CMake_SOURCE_DIR@/Tests/CTestTestParallel")
+SET(CTEST_BINARY_DIRECTORY              "@CMake_BINARY_DIR@/Tests/CTestTestParallel")
+SET(CTEST_CVS_COMMAND                   "@CVSCOMMAND@")
+SET(CTEST_CMAKE_GENERATOR               "@CMAKE_TEST_GENERATOR@")
+SET(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+SET(CTEST_MEMORYCHECK_COMMAND           "@MEMORYCHECK_COMMAND@")
+SET(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "@MEMORYCHECK_SUPPRESSIONS_FILE@")
+SET(CTEST_MEMORYCHECK_COMMAND_OPTIONS   "@MEMORYCHECK_COMMAND_OPTIONS@")
+SET(CTEST_COVERAGE_COMMAND              "@COVERAGE_COMMAND@")
+SET(CTEST_NOTES_FILES                   "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}")
+
+CTEST_START(Experimental)
+CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 4)
+
+SET(CTEST_DROP_METHOD "@protocol@")
+SET(CTEST_DROP_SITE "@server@")
+SET(CTEST_DROP_LOCATION "@path@/submit.php?project=PublicDashboard")
+
+CTEST_SUBMIT(RETRY_DELAY 3 RETRY_COUNT 2 INTERNAL_TEST_CHECKSUM RETURN_VALUE res)