Implement file(UPLOAD (#11286)

Including documentation and testing, of course.

Source/cmFileCommand.cxx

@@ -75,6 +75,10 @@ bool cmFileCommand
     {
     return this->HandleDownloadCommand(args);
     }
+  else if ( subCommand == "UPLOAD" )
+    {
+    return this->HandleUploadCommand(args);
+    }
   else if ( subCommand == "READ" )
     {
     return this->HandleReadCommand(args);
@@ -2432,13 +2436,17 @@ bool cmFileCommand::HandleCMakePathCommand(std::vector<std::string>
   this->Makefile->AddDefinition(var, value.c_str());
   return true;
 }
+
+
 #if defined(CMAKE_BUILD_WITH_CMAKE)
 
-// Stuff for curl download
+// Stuff for curl download/upload
 typedef std::vector<char> cmFileCommandVectorOfChar;
-namespace{
+
+namespace {
+
   size_t
-  cmFileCommandWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
+  cmWriteToFileCallback(void *ptr, size_t size, size_t nmemb,
                         void *data)
     {
     register int realsize = (int)(size * nmemb);
@@ -2449,6 +2457,19 @@ namespace{
     }
 
 
+  size_t
+  cmWriteToMemoryCallback(void *ptr, size_t size, size_t nmemb,
+                          void *data)
+    {
+    register int realsize = (int)(size * nmemb);
+    cmFileCommandVectorOfChar *vec
+      = static_cast<cmFileCommandVectorOfChar*>(data);
+    const char* chPtr = static_cast<char*>(ptr);
+    vec->insert(vec->end(), chPtr, chPtr + realsize);
+    return realsize;
+    }
+
+
   static size_t
   cmFileCommandCurlDebugCallback(CURL *, curl_infotype, char *chPtr,
                                  size_t size, void *data)
@@ -2456,7 +2477,6 @@ namespace{
     cmFileCommandVectorOfChar *vec
       = static_cast<cmFileCommandVectorOfChar*>(data);
     vec->insert(vec->end(), chPtr, chPtr + size);
-
     return size;
     }
 
@@ -2464,21 +2484,18 @@ namespace{
   class cURLProgressHelper
   {
   public:
-    cURLProgressHelper(cmFileCommand *fc)
+    cURLProgressHelper(cmFileCommand *fc, const char *text)
       {
       this->CurrentPercentage = -1;
       this->FileCommand = fc;
+      this->Text = text;
       }
 
     bool UpdatePercentage(double value, double total, std::string &status)
       {
       int OldPercentage = this->CurrentPercentage;
 
-      if (0.0 == total)
+      if (total > 0.0)
-        {
-        this->CurrentPercentage = 100;
-        }
-      else
         {
         this->CurrentPercentage = static_cast<int>(value/total*100.0 + 0.5);
         }
@@ -2488,7 +2505,8 @@ namespace{
       if (updated)
         {
         cmOStringStream oss;
-        oss << "[download " << this->CurrentPercentage << "% complete]";
+        oss << "[" << this->Text << " " << this->CurrentPercentage
+            << "% complete]";
         status = oss.str();
         }
 
@@ -2503,11 +2521,12 @@ namespace{
   private:
     int CurrentPercentage;
     cmFileCommand *FileCommand;
+    std::string Text;
   };
 
 
   static int
-  cmFileCommandCurlProgressCallback(void *clientp,
+  cmFileDownloadProgressCallback(void *clientp,
                                  double dltotal, double dlnow,
                                  double ultotal, double ulnow)
     {
@@ -2527,11 +2546,32 @@ namespace{
 
     return 0;
     }
+
+
+  static int
+  cmFileUploadProgressCallback(void *clientp,
+                               double dltotal, double dlnow,
+                               double ultotal, double ulnow)
+    {
+    cURLProgressHelper *helper =
+    reinterpret_cast<cURLProgressHelper *>(clientp);
+
+    static_cast<void>(dltotal);
+    static_cast<void>(dlnow);
+
+    std::string status;
+    if (helper->UpdatePercentage(ulnow, ultotal, status))
+      {
+      cmFileCommand *fc = helper->GetFileCommand();
+      cmMakefile *mf = fc->GetMakefile();
+      mf->DisplayStatus(status.c_str(), -1);
+      }
+
+    return 0;
+    }
 }
 
-#endif
 
-#if defined(CMAKE_BUILD_WITH_CMAKE)
 namespace {
 
   class cURLEasyGuard
@@ -2563,9 +2603,18 @@ namespace {
 #endif
 
 
+#define check_curl_result(result, errstr) \
+  if (result != CURLE_OK)                 \
+    {                                     \
+    std::string e(errstr);                \
+    e += ::curl_easy_strerror(result);    \
+    this->SetError(e.c_str());            \
+    return false;                         \
+    }
+
+
 bool
-cmFileCommand::HandleDownloadCommand(std::vector<std::string>
+cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
-                                     const& args)
 {
 #if defined(CMAKE_BUILD_WITH_CMAKE)
   std::vector<std::string>::const_iterator i = args.begin();
@@ -2704,88 +2753,37 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
   cURLEasyGuard g_curl(curl);
 
   ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-  if (res != CURLE_OK)
+  check_curl_result(res, "DOWNLOAD cannot set url: ");
-    {
-    std::string e = "DOWNLOAD cannot set url: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-                           cmFileCommandWriteMemoryCallback);
+                           cmWriteToFileCallback);
-  if (res != CURLE_OK)
+  check_curl_result(res, "DOWNLOAD cannot set write function: ");
-    {
-    std::string e = "DOWNLOAD cannot set write function: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
                            cmFileCommandCurlDebugCallback);
-  if (res != CURLE_OK)
+  check_curl_result(res, "DOWNLOAD cannot set debug function: ");
-    {
-    std::string e = "DOWNLOAD cannot set debug function: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   cmFileCommandVectorOfChar chunkDebug;
 
   res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fout);
-
+  check_curl_result(res, "DOWNLOAD cannot set write data: ");
-  if (res != CURLE_OK)
-    {
-    std::string e = "DOWNLOAD cannot set write data: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug);
-  if (res != CURLE_OK)
+  check_curl_result(res, "DOWNLOAD cannot set debug data: ");
-    {
-    std::string e = "DOWNLOAD cannot set debug data: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-  if (res != CURLE_OK)
+  check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
-    {
-    std::string e = "DOWNLOAD cannot set follow-redirect option: ";
-    e += ::curl_easy_strerror(res);
-    this->SetError(e.c_str());
-    return false;
-    }
 
   if(verboseLog.size())
     {
     res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
-
+    check_curl_result(res, "DOWNLOAD cannot set verbose: ");
-    if (res != CURLE_OK)
-      {
-      std::string e = "DOWNLOAD cannot set verbose: ";
-      e += ::curl_easy_strerror(res);
-      this->SetError(e.c_str());
-      return false;
-      }
     }
 
   if(timeout > 0)
     {
     res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout );
-
+    check_curl_result(res, "DOWNLOAD cannot set timeout: ");
-    if (res != CURLE_OK)
-      {
-      std::string e = "DOWNLOAD cannot set timeout: ";
-      e += ::curl_easy_strerror(res);
-      this->SetError(e.c_str());
-      return false;
-      }
     }
 
   // Need the progress helper's scope to last through the duration of
@@ -2793,39 +2791,20 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
   // scope intentionally, rather than inside the "if(showProgress)"
   // block...
   //
-  cURLProgressHelper helper(this);
+  cURLProgressHelper helper(this, "download");
 
   if(showProgress)
     {
-    res = ::curl_easy_setopt(curl,
+    res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
-      CURLOPT_NOPROGRESS, 0);
+    check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
-    if (res != CURLE_OK)
-      {
-      std::string e = "DOWNLOAD cannot set noprogress value: ";
-      e += ::curl_easy_strerror(res);
-      this->SetError(e.c_str());
-      return false;
-      }
 
     res = ::curl_easy_setopt(curl,
-      CURLOPT_PROGRESSFUNCTION, cmFileCommandCurlProgressCallback);
+      CURLOPT_PROGRESSFUNCTION, cmFileDownloadProgressCallback);
-    if (res != CURLE_OK)
+    check_curl_result(res, "DOWNLOAD cannot set progress function: ");
-      {
-      std::string e = "DOWNLOAD cannot set progress function: ";
-      e += ::curl_easy_strerror(res);
-      this->SetError(e.c_str());
-      return false;
-      }
 
     res = ::curl_easy_setopt(curl,
       CURLOPT_PROGRESSDATA, reinterpret_cast<void*>(&helper));
-    if (res != CURLE_OK)
+    check_curl_result(res, "DOWNLOAD cannot set progress data: ");
-      {
-      std::string e = "DOWNLOAD cannot set progress data: ";
-      e += ::curl_easy_strerror(res);
-      this->SetError(e.c_str());
-      return false;
-      }
     }
 
   res = ::curl_easy_perform(curl);
@@ -2901,3 +2880,219 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
   return false;
 #endif
 }
+
+
+bool
+cmFileCommand::HandleUploadCommand(std::vector<std::string> const& args)
+{
+#if defined(CMAKE_BUILD_WITH_CMAKE)
+  if(args.size() < 3)
+    {
+    this->SetError("UPLOAD must be called with at least three arguments.");
+    return false;
+    }
+  std::vector<std::string>::const_iterator i = args.begin();
+  ++i;
+  std::string filename = *i;
+  ++i;
+  std::string url = *i;
+  ++i;
+
+  long timeout = 0;
+  std::string logVar;
+  std::string statusVar;
+  bool showProgress = false;
+
+  while(i != args.end())
+    {
+    if(*i == "TIMEOUT")
+      {
+      ++i;
+      if(i != args.end())
+        {
+        timeout = atol(i->c_str());
+        }
+      else
+        {
+        this->SetError("UPLOAD missing time for TIMEOUT.");
+        return false;
+        }
+      }
+    else if(*i == "LOG")
+      {
+      ++i;
+      if( i == args.end())
+        {
+        this->SetError("UPLOAD missing VAR for LOG.");
+        return false;
+        }
+      logVar = *i;
+      }
+    else if(*i == "STATUS")
+      {
+      ++i;
+      if( i == args.end())
+        {
+        this->SetError("UPLOAD missing VAR for STATUS.");
+        return false;
+        }
+      statusVar = *i;
+      }
+    else if(*i == "SHOW_PROGRESS")
+      {
+      showProgress = true;
+      }
+
+    ++i;
+    }
+
+  // Open file for reading:
+  //
+  FILE *fin = fopen(filename.c_str(), "rb");
+  if(!fin)
+    {
+    std::string errStr = "UPLOAD cannot open file '";
+    errStr += filename + "' for reading.";
+    this->SetError(errStr.c_str());
+    return false;
+    }
+
+  struct stat st;
+  if(::stat(filename.c_str(), &st))
+    {
+    std::string errStr = "UPLOAD cannot stat file '";
+    errStr += filename + "'.";
+    this->SetError(errStr.c_str());
+    return false;
+    }
+
+  ::CURL *curl;
+  ::curl_global_init(CURL_GLOBAL_DEFAULT);
+  curl = ::curl_easy_init();
+  if(!curl)
+    {
+    this->SetError("UPLOAD error initializing curl.");
+    return false;
+    }
+
+  cURLEasyGuard g_curl(curl);
+
+  // enable HTTP ERROR parsing
+  ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+
+  // enable uploading
+  res = ::curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
+  check_curl_result(res, "UPLOAD cannot set upload flag: ");
+
+  res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  check_curl_result(res, "UPLOAD cannot set url: ");
+
+  res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+                           cmWriteToMemoryCallback);
+  check_curl_result(res, "UPLOAD cannot set write function: ");
+
+  res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
+                           cmFileCommandCurlDebugCallback);
+  check_curl_result(res, "UPLOAD cannot set debug function: ");
+
+  cmFileCommandVectorOfChar chunkResponse;
+  cmFileCommandVectorOfChar chunkDebug;
+
+  res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunkResponse);
+  check_curl_result(res, "UPLOAD cannot set write data: ");
+
+  res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug);
+  check_curl_result(res, "UPLOAD cannot set debug data: ");
+
+  res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+  check_curl_result(res, "UPLOAD cannot set follow-redirect option: ");
+
+  if(logVar.size())
+    {
+    res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+    check_curl_result(res, "UPLOAD cannot set verbose: ");
+    }
+
+  if(timeout > 0)
+    {
+    res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout );
+    check_curl_result(res, "UPLOAD cannot set timeout: ");
+    }
+
+  // Need the progress helper's scope to last through the duration of
+  // the curl_easy_perform call... so this object is declared at function
+  // scope intentionally, rather than inside the "if(showProgress)"
+  // block...
+  //
+  cURLProgressHelper helper(this, "upload");
+
+  if(showProgress)
+    {
+    res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+    check_curl_result(res, "UPLOAD cannot set noprogress value: ");
+
+    res = ::curl_easy_setopt(curl,
+      CURLOPT_PROGRESSFUNCTION, cmFileUploadProgressCallback);
+    check_curl_result(res, "UPLOAD cannot set progress function: ");
+
+    res = ::curl_easy_setopt(curl,
+      CURLOPT_PROGRESSDATA, reinterpret_cast<void*>(&helper));
+    check_curl_result(res, "UPLOAD cannot set progress data: ");
+    }
+
+  // now specify which file to upload
+  res = ::curl_easy_setopt(curl, CURLOPT_INFILE, fin);
+  check_curl_result(res, "UPLOAD cannot set input file: ");
+
+  // and give the size of the upload (optional)
+  res = ::curl_easy_setopt(curl, CURLOPT_INFILESIZE, static_cast<long>(st.st_size));
+  check_curl_result(res, "UPLOAD cannot set input file size: ");
+
+  res = ::curl_easy_perform(curl);
+
+  /* always cleanup */
+  g_curl.release();
+  ::curl_easy_cleanup(curl);
+
+  if(statusVar.size())
+    {
+    cmOStringStream result;
+    result << (int)res << ";\"" << ::curl_easy_strerror(res) << "\"";
+    this->Makefile->AddDefinition(statusVar.c_str(),
+                                  result.str().c_str());
+    }
+
+  ::curl_global_cleanup();
+
+  fclose(fin);
+  fin = NULL;
+
+  if(logVar.size())
+    {
+    std::string log;
+
+    if(chunkResponse.size())
+      {
+      chunkResponse.push_back(0);
+      log += "Response:\n";
+      log += &*chunkResponse.begin();
+      log += "\n";
+      }
+
+    if(chunkDebug.size())
+      {
+      chunkDebug.push_back(0);
+      log += "Debug:\n";
+      log += &*chunkDebug.begin();
+      log += "\n";
+      }
+
+    this->Makefile->AddDefinition(logVar.c_str(), log.c_str());
+    }
+
+  return true;
+#else
+  this->SetError("UPLOAD not supported by bootstrap cmake.");
+  return false;
+#endif
+}

Source/cmFileCommand.h

@@ -82,6 +82,8 @@ public:
       "  file(TO_NATIVE_PATH path result)\n"
       "  file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log]\n"
       "       [EXPECTED_MD5 sum] [SHOW_PROGRESS])\n"
+      "  file(UPLOAD filename url [TIMEOUT timeout] [STATUS status]\n"
+      "       [LOG log] [SHOW_PROGRESS])\n"
       "WRITE will write a message into a file called 'filename'. It "
       "overwrites the file if it already exists, and creates the file "
       "if it does not exist.\n"
@@ -165,6 +167,18 @@ public:
       "If SHOW_PROGRESS is specified, progress information will be printed "
       "as status messages until the operation is complete."
       "\n"
+      "UPLOAD will upload the given file to the given URL. "
+      "If LOG var is specified a log of the upload will be put in var. "
+      "If STATUS var is specified the status of the operation will"
+      " be put in var. The status is returned in a list of length 2. "
+      "The first element is the numeric return value for the operation, "
+      "and the second element is a string value for the error. A 0 "
+      "numeric error means no error in the operation. "
+      "If TIMEOUT time is specified, the operation will "
+      "timeout after time seconds, time should be specified as an integer. "
+      "If SHOW_PROGRESS is specified, progress information will be printed "
+      "as status messages until the operation is complete."
+      "\n"
       "The file() command also provides COPY and INSTALL signatures:\n"
       "  file(<COPY|INSTALL> files... DESTINATION <dir>\n"
       "       [FILE_PERMISSIONS permissions...]\n"
@@ -223,6 +237,7 @@ protected:
   bool HandleCopyCommand(std::vector<std::string> const& args);
   bool HandleInstallCommand(std::vector<std::string> const& args);
   bool HandleDownloadCommand(std::vector<std::string> const& args);
+  bool HandleUploadCommand(std::vector<std::string> const& args);
 };
 
 

Tests/CMakeTests/CMakeLists.txt

@@ -33,6 +33,8 @@ set_property(TEST CMake.FileDownload PROPERTY
   PASS_REGULAR_EXPRESSION "file already exists with expected MD5 sum"
   )
 
+AddCMakeTest(FileUpload "")
+
 if(HAVE_ELF_H)
   AddCMakeTest(ELF "")
 endif()

Tests/CMakeTests/FileTest.cmake.in

@@ -27,7 +27,7 @@ check_cmake_test(File
 # Also execute each test listed in FileTestScript.cmake:
 #
 set(scriptname "@CMAKE_CURRENT_SOURCE_DIR@/FileTestScript.cmake")
-set(number_of_tests_expected 56)
+set(number_of_tests_expected 62)
 
 include("@CMAKE_CURRENT_SOURCE_DIR@/ExecuteScriptTests.cmake")
 execute_all_script_tests(${scriptname} number_of_tests_executed)

Tests/CMakeTests/FileTestScript.cmake

@@ -201,6 +201,26 @@ elseif(testname STREQUAL download_with_bogus_protocol) # pass
   message("l='${l}'")
   message("s='${s}'")
 
+elseif(testname STREQUAL upload_wrong_number_of_args) # fail
+  file(UPLOAD ./ffff)
+
+elseif(testname STREQUAL upload_missing_time) # fail
+  file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT)
+
+elseif(testname STREQUAL upload_missing_log_var) # fail
+  file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT 2 LOG)
+
+elseif(testname STREQUAL upload_missing_status_var) # fail
+  file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT 2 LOG l STATUS)
+
+elseif(testname STREQUAL upload_file_that_doesnt_exist) # fail
+  file(UPLOAD ./ffff zzzz://bogus/ffff)
+
+elseif(testname STREQUAL upload_with_bogus_protocol) # pass
+  file(UPLOAD ${CMAKE_CURRENT_LIST_FILE} zzzz://bogus/ffff TIMEOUT 2 LOG l STATUS s)
+  message("l='${l}'")
+  message("s='${s}'")
+
 else() # fail
   message(FATAL_ERROR "testname='${testname}' - error: no such test in '${CMAKE_CURRENT_LIST_FILE}'")
 

Tests/CMakeTests/FileUploadTest.cmake.in

@@ -0,0 +1,49 @@
+file(REMOVE_RECURSE "@CMAKE_CURRENT_BINARY_DIR@/uploads")
+
+if(EXISTS "@CMAKE_CURRENT_BINARY_DIR@/uploads/file1.png")
+  message(FATAL_ERROR "error: file1.png exists - should have been deleted")
+endif()
+if(EXISTS "@CMAKE_CURRENT_BINARY_DIR@/uploads/file2.png")
+  message(FATAL_ERROR "error: file2.png exists - should have been deleted")
+endif()
+
+file(MAKE_DIRECTORY "@CMAKE_CURRENT_BINARY_DIR@/uploads")
+
+set(filename "@CMAKE_CURRENT_SOURCE_DIR@/FileDownloadInput.png")
+set(urlbase "file://@CMAKE_CURRENT_BINARY_DIR@/uploads")
+
+message(STATUS "FileUpload:1")
+file(UPLOAD
+  ${filename}
+  ${urlbase}/file1.png
+  TIMEOUT 2
+  )
+
+message(STATUS "FileUpload:2")
+file(UPLOAD
+  ${filename}
+  ${urlbase}/file2.png
+  STATUS status
+  LOG log
+  SHOW_PROGRESS
+  )
+
+execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum
+  "@CMAKE_CURRENT_BINARY_DIR@/uploads/file1.png"
+  OUTPUT_VARIABLE sum1
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(NOT sum1 MATCHES "^d16778650db435bda3a8c3435c3ff5d1  .*/uploads/file1.png$")
+  message(FATAL_ERROR "file1.png did not upload correctly (sum1='${sum1}')")
+endif()
+
+execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum
+  "@CMAKE_CURRENT_BINARY_DIR@/uploads/file2.png"
+  OUTPUT_VARIABLE sum2
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(NOT sum2 MATCHES "^d16778650db435bda3a8c3435c3ff5d1  .*/uploads/file2.png$")
+  message(FATAL_ERROR "file2.png did not upload correctly (sum2='${sum2}')")
+endif()
+
+message(STATUS "log='${log}'")
+message(STATUS "status='${status}'")
+message(STATUS "DONE")