From 963bebcc174411e1087af31cc5054422b0080e76 Mon Sep 17 00:00:00 2001 From: David Cole Date: Wed, 23 Feb 2011 13:53:53 -0500 Subject: [PATCH 1/2] Implement file(UPLOAD (#11286) Including documentation and testing, of course. --- Source/cmFileCommand.cxx | 417 +++++++++++++++++------ Source/cmFileCommand.h | 15 + Tests/CMakeTests/CMakeLists.txt | 2 + Tests/CMakeTests/FileTest.cmake.in | 2 +- Tests/CMakeTests/FileTestScript.cmake | 20 ++ Tests/CMakeTests/FileUploadTest.cmake.in | 49 +++ 6 files changed, 393 insertions(+), 112 deletions(-) create mode 100644 Tests/CMakeTests/FileUploadTest.cmake.in diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 37fced2db..8849a470c 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -75,6 +75,10 @@ bool cmFileCommand { return this->HandleDownloadCommand(args); } + else if ( subCommand == "UPLOAD" ) + { + return this->HandleUploadCommand(args); + } else if ( subCommand == "READ" ) { return this->HandleReadCommand(args); @@ -2432,53 +2436,66 @@ bool cmFileCommand::HandleCMakePathCommand(std::vector this->Makefile->AddDefinition(var, value.c_str()); return true; } + + #if defined(CMAKE_BUILD_WITH_CMAKE) -// Stuff for curl download +// Stuff for curl download/upload typedef std::vector cmFileCommandVectorOfChar; -namespace{ + +namespace { + size_t - cmFileCommandWriteMemoryCallback(void *ptr, size_t size, size_t nmemb, - void *data) - { + cmWriteToFileCallback(void *ptr, size_t size, size_t nmemb, + void *data) + { register int realsize = (int)(size * nmemb); std::ofstream* fout = static_cast(data); const char* chPtr = static_cast(ptr); fout->write(chPtr, realsize); return realsize; - } + } + + + size_t + cmWriteToMemoryCallback(void *ptr, size_t size, size_t nmemb, + void *data) + { + register int realsize = (int)(size * nmemb); + cmFileCommandVectorOfChar *vec + = static_cast(data); + const char* chPtr = static_cast(ptr); + vec->insert(vec->end(), chPtr, chPtr + realsize); + return realsize; + } static size_t cmFileCommandCurlDebugCallback(CURL *, curl_infotype, char *chPtr, - size_t size, void *data) - { + size_t size, void *data) + { cmFileCommandVectorOfChar *vec = static_cast(data); vec->insert(vec->end(), chPtr, chPtr + size); - return size; - } + } class cURLProgressHelper { public: - cURLProgressHelper(cmFileCommand *fc) + cURLProgressHelper(cmFileCommand *fc, const char *text) { this->CurrentPercentage = -1; this->FileCommand = fc; + this->Text = text; } bool UpdatePercentage(double value, double total, std::string &status) { int OldPercentage = this->CurrentPercentage; - if (0.0 == total) - { - this->CurrentPercentage = 100; - } - else + if (total > 0.0) { this->CurrentPercentage = static_cast(value/total*100.0 + 0.5); } @@ -2488,7 +2505,8 @@ namespace{ if (updated) { cmOStringStream oss; - oss << "[download " << this->CurrentPercentage << "% complete]"; + oss << "[" << this->Text << " " << this->CurrentPercentage + << "% complete]"; status = oss.str(); } @@ -2503,14 +2521,15 @@ namespace{ private: int CurrentPercentage; cmFileCommand *FileCommand; + std::string Text; }; static int - cmFileCommandCurlProgressCallback(void *clientp, - double dltotal, double dlnow, - double ultotal, double ulnow) - { + cmFileDownloadProgressCallback(void *clientp, + double dltotal, double dlnow, + double ultotal, double ulnow) + { cURLProgressHelper *helper = reinterpret_cast(clientp); @@ -2526,12 +2545,33 @@ namespace{ } return 0; - } + } + + + static int + cmFileUploadProgressCallback(void *clientp, + double dltotal, double dlnow, + double ultotal, double ulnow) + { + cURLProgressHelper *helper = + reinterpret_cast(clientp); + + static_cast(dltotal); + static_cast(dlnow); + + std::string status; + if (helper->UpdatePercentage(ulnow, ultotal, status)) + { + cmFileCommand *fc = helper->GetFileCommand(); + cmMakefile *mf = fc->GetMakefile(); + mf->DisplayStatus(status.c_str(), -1); + } + + return 0; + } } -#endif -#if defined(CMAKE_BUILD_WITH_CMAKE) namespace { class cURLEasyGuard @@ -2563,9 +2603,18 @@ namespace { #endif +#define check_curl_result(result, errstr) \ + if (result != CURLE_OK) \ + { \ + std::string e(errstr); \ + e += ::curl_easy_strerror(result); \ + this->SetError(e.c_str()); \ + return false; \ + } + + bool -cmFileCommand::HandleDownloadCommand(std::vector - const& args) +cmFileCommand::HandleDownloadCommand(std::vector const& args) { #if defined(CMAKE_BUILD_WITH_CMAKE) std::vector::const_iterator i = args.begin(); @@ -2704,88 +2753,37 @@ cmFileCommand::HandleDownloadCommand(std::vector cURLEasyGuard g_curl(curl); ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set url: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set url: "); res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, - cmFileCommandWriteMemoryCallback); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set write function: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + cmWriteToFileCallback); + check_curl_result(res, "DOWNLOAD cannot set write function: "); res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, cmFileCommandCurlDebugCallback); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set debug function: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set debug function: "); cmFileCommandVectorOfChar chunkDebug; res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fout); - - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set write data: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set write data: "); res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set debug data: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set debug data: "); res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set follow-redirect option: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: "); if(verboseLog.size()) { res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set verbose: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set verbose: "); } if(timeout > 0) { res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout ); - - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set timeout: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set timeout: "); } // Need the progress helper's scope to last through the duration of @@ -2793,39 +2791,20 @@ cmFileCommand::HandleDownloadCommand(std::vector // scope intentionally, rather than inside the "if(showProgress)" // block... // - cURLProgressHelper helper(this); + cURLProgressHelper helper(this, "download"); if(showProgress) { - res = ::curl_easy_setopt(curl, - CURLOPT_NOPROGRESS, 0); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set noprogress value: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + check_curl_result(res, "DOWNLOAD cannot set noprogress value: "); res = ::curl_easy_setopt(curl, - CURLOPT_PROGRESSFUNCTION, cmFileCommandCurlProgressCallback); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set progress function: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + CURLOPT_PROGRESSFUNCTION, cmFileDownloadProgressCallback); + check_curl_result(res, "DOWNLOAD cannot set progress function: "); res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, reinterpret_cast(&helper)); - if (res != CURLE_OK) - { - std::string e = "DOWNLOAD cannot set progress data: "; - e += ::curl_easy_strerror(res); - this->SetError(e.c_str()); - return false; - } + check_curl_result(res, "DOWNLOAD cannot set progress data: "); } res = ::curl_easy_perform(curl); @@ -2901,3 +2880,219 @@ cmFileCommand::HandleDownloadCommand(std::vector return false; #endif } + + +bool +cmFileCommand::HandleUploadCommand(std::vector const& args) +{ +#if defined(CMAKE_BUILD_WITH_CMAKE) + if(args.size() < 3) + { + this->SetError("UPLOAD must be called with at least three arguments."); + return false; + } + std::vector::const_iterator i = args.begin(); + ++i; + std::string filename = *i; + ++i; + std::string url = *i; + ++i; + + long timeout = 0; + std::string logVar; + std::string statusVar; + bool showProgress = false; + + while(i != args.end()) + { + if(*i == "TIMEOUT") + { + ++i; + if(i != args.end()) + { + timeout = atol(i->c_str()); + } + else + { + this->SetError("UPLOAD missing time for TIMEOUT."); + return false; + } + } + else if(*i == "LOG") + { + ++i; + if( i == args.end()) + { + this->SetError("UPLOAD missing VAR for LOG."); + return false; + } + logVar = *i; + } + else if(*i == "STATUS") + { + ++i; + if( i == args.end()) + { + this->SetError("UPLOAD missing VAR for STATUS."); + return false; + } + statusVar = *i; + } + else if(*i == "SHOW_PROGRESS") + { + showProgress = true; + } + + ++i; + } + + // Open file for reading: + // + FILE *fin = fopen(filename.c_str(), "rb"); + if(!fin) + { + std::string errStr = "UPLOAD cannot open file '"; + errStr += filename + "' for reading."; + this->SetError(errStr.c_str()); + return false; + } + + struct stat st; + if(::stat(filename.c_str(), &st)) + { + std::string errStr = "UPLOAD cannot stat file '"; + errStr += filename + "'."; + this->SetError(errStr.c_str()); + return false; + } + + ::CURL *curl; + ::curl_global_init(CURL_GLOBAL_DEFAULT); + curl = ::curl_easy_init(); + if(!curl) + { + this->SetError("UPLOAD error initializing curl."); + return false; + } + + cURLEasyGuard g_curl(curl); + + // enable HTTP ERROR parsing + ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + + // enable uploading + res = ::curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + check_curl_result(res, "UPLOAD cannot set upload flag: "); + + res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + check_curl_result(res, "UPLOAD cannot set url: "); + + res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + cmWriteToMemoryCallback); + check_curl_result(res, "UPLOAD cannot set write function: "); + + res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, + cmFileCommandCurlDebugCallback); + check_curl_result(res, "UPLOAD cannot set debug function: "); + + cmFileCommandVectorOfChar chunkResponse; + cmFileCommandVectorOfChar chunkDebug; + + res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunkResponse); + check_curl_result(res, "UPLOAD cannot set write data: "); + + res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug); + check_curl_result(res, "UPLOAD cannot set debug data: "); + + res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + check_curl_result(res, "UPLOAD cannot set follow-redirect option: "); + + if(logVar.size()) + { + res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + check_curl_result(res, "UPLOAD cannot set verbose: "); + } + + if(timeout > 0) + { + res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout ); + check_curl_result(res, "UPLOAD cannot set timeout: "); + } + + // Need the progress helper's scope to last through the duration of + // the curl_easy_perform call... so this object is declared at function + // scope intentionally, rather than inside the "if(showProgress)" + // block... + // + cURLProgressHelper helper(this, "upload"); + + if(showProgress) + { + res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + check_curl_result(res, "UPLOAD cannot set noprogress value: "); + + res = ::curl_easy_setopt(curl, + CURLOPT_PROGRESSFUNCTION, cmFileUploadProgressCallback); + check_curl_result(res, "UPLOAD cannot set progress function: "); + + res = ::curl_easy_setopt(curl, + CURLOPT_PROGRESSDATA, reinterpret_cast(&helper)); + check_curl_result(res, "UPLOAD cannot set progress data: "); + } + + // now specify which file to upload + res = ::curl_easy_setopt(curl, CURLOPT_INFILE, fin); + check_curl_result(res, "UPLOAD cannot set input file: "); + + // and give the size of the upload (optional) + res = ::curl_easy_setopt(curl, CURLOPT_INFILESIZE, static_cast(st.st_size)); + check_curl_result(res, "UPLOAD cannot set input file size: "); + + res = ::curl_easy_perform(curl); + + /* always cleanup */ + g_curl.release(); + ::curl_easy_cleanup(curl); + + if(statusVar.size()) + { + cmOStringStream result; + result << (int)res << ";\"" << ::curl_easy_strerror(res) << "\""; + this->Makefile->AddDefinition(statusVar.c_str(), + result.str().c_str()); + } + + ::curl_global_cleanup(); + + fclose(fin); + fin = NULL; + + if(logVar.size()) + { + std::string log; + + if(chunkResponse.size()) + { + chunkResponse.push_back(0); + log += "Response:\n"; + log += &*chunkResponse.begin(); + log += "\n"; + } + + if(chunkDebug.size()) + { + chunkDebug.push_back(0); + log += "Debug:\n"; + log += &*chunkDebug.begin(); + log += "\n"; + } + + this->Makefile->AddDefinition(logVar.c_str(), log.c_str()); + } + + return true; +#else + this->SetError("UPLOAD not supported by bootstrap cmake."); + return false; +#endif +} diff --git a/Source/cmFileCommand.h b/Source/cmFileCommand.h index b11dcde4d..1b6dbbfb1 100644 --- a/Source/cmFileCommand.h +++ b/Source/cmFileCommand.h @@ -82,6 +82,8 @@ public: " file(TO_NATIVE_PATH path result)\n" " file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log]\n" " [EXPECTED_MD5 sum] [SHOW_PROGRESS])\n" + " file(UPLOAD filename url [TIMEOUT timeout] [STATUS status]\n" + " [LOG log] [SHOW_PROGRESS])\n" "WRITE will write a message into a file called 'filename'. It " "overwrites the file if it already exists, and creates the file " "if it does not exist.\n" @@ -165,6 +167,18 @@ public: "If SHOW_PROGRESS is specified, progress information will be printed " "as status messages until the operation is complete." "\n" + "UPLOAD will upload the given file to the given URL. " + "If LOG var is specified a log of the upload will be put in var. " + "If STATUS var is specified the status of the operation will" + " be put in var. The status is returned in a list of length 2. " + "The first element is the numeric return value for the operation, " + "and the second element is a string value for the error. A 0 " + "numeric error means no error in the operation. " + "If TIMEOUT time is specified, the operation will " + "timeout after time seconds, time should be specified as an integer. " + "If SHOW_PROGRESS is specified, progress information will be printed " + "as status messages until the operation is complete." + "\n" "The file() command also provides COPY and INSTALL signatures:\n" " file( files... DESTINATION \n" " [FILE_PERMISSIONS permissions...]\n" @@ -223,6 +237,7 @@ protected: bool HandleCopyCommand(std::vector const& args); bool HandleInstallCommand(std::vector const& args); bool HandleDownloadCommand(std::vector const& args); + bool HandleUploadCommand(std::vector const& args); }; diff --git a/Tests/CMakeTests/CMakeLists.txt b/Tests/CMakeTests/CMakeLists.txt index 8fd52dfd3..cceef3bcf 100644 --- a/Tests/CMakeTests/CMakeLists.txt +++ b/Tests/CMakeTests/CMakeLists.txt @@ -33,6 +33,8 @@ set_property(TEST CMake.FileDownload PROPERTY PASS_REGULAR_EXPRESSION "file already exists with expected MD5 sum" ) +AddCMakeTest(FileUpload "") + if(HAVE_ELF_H) AddCMakeTest(ELF "") endif() diff --git a/Tests/CMakeTests/FileTest.cmake.in b/Tests/CMakeTests/FileTest.cmake.in index 721f311f9..b6dcaa6d1 100644 --- a/Tests/CMakeTests/FileTest.cmake.in +++ b/Tests/CMakeTests/FileTest.cmake.in @@ -27,7 +27,7 @@ check_cmake_test(File # Also execute each test listed in FileTestScript.cmake: # set(scriptname "@CMAKE_CURRENT_SOURCE_DIR@/FileTestScript.cmake") -set(number_of_tests_expected 56) +set(number_of_tests_expected 62) include("@CMAKE_CURRENT_SOURCE_DIR@/ExecuteScriptTests.cmake") execute_all_script_tests(${scriptname} number_of_tests_executed) diff --git a/Tests/CMakeTests/FileTestScript.cmake b/Tests/CMakeTests/FileTestScript.cmake index c4e27bd27..9a4356973 100644 --- a/Tests/CMakeTests/FileTestScript.cmake +++ b/Tests/CMakeTests/FileTestScript.cmake @@ -201,6 +201,26 @@ elseif(testname STREQUAL download_with_bogus_protocol) # pass message("l='${l}'") message("s='${s}'") +elseif(testname STREQUAL upload_wrong_number_of_args) # fail + file(UPLOAD ./ffff) + +elseif(testname STREQUAL upload_missing_time) # fail + file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT) + +elseif(testname STREQUAL upload_missing_log_var) # fail + file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT 2 LOG) + +elseif(testname STREQUAL upload_missing_status_var) # fail + file(UPLOAD ./ffff zzzz://bogus/ffff TIMEOUT 2 LOG l STATUS) + +elseif(testname STREQUAL upload_file_that_doesnt_exist) # fail + file(UPLOAD ./ffff zzzz://bogus/ffff) + +elseif(testname STREQUAL upload_with_bogus_protocol) # pass + file(UPLOAD ${CMAKE_CURRENT_LIST_FILE} zzzz://bogus/ffff TIMEOUT 2 LOG l STATUS s) + message("l='${l}'") + message("s='${s}'") + else() # fail message(FATAL_ERROR "testname='${testname}' - error: no such test in '${CMAKE_CURRENT_LIST_FILE}'") diff --git a/Tests/CMakeTests/FileUploadTest.cmake.in b/Tests/CMakeTests/FileUploadTest.cmake.in new file mode 100644 index 000000000..8577aefd2 --- /dev/null +++ b/Tests/CMakeTests/FileUploadTest.cmake.in @@ -0,0 +1,49 @@ +file(REMOVE_RECURSE "@CMAKE_CURRENT_BINARY_DIR@/uploads") + +if(EXISTS "@CMAKE_CURRENT_BINARY_DIR@/uploads/file1.png") + message(FATAL_ERROR "error: file1.png exists - should have been deleted") +endif() +if(EXISTS "@CMAKE_CURRENT_BINARY_DIR@/uploads/file2.png") + message(FATAL_ERROR "error: file2.png exists - should have been deleted") +endif() + +file(MAKE_DIRECTORY "@CMAKE_CURRENT_BINARY_DIR@/uploads") + +set(filename "@CMAKE_CURRENT_SOURCE_DIR@/FileDownloadInput.png") +set(urlbase "file://@CMAKE_CURRENT_BINARY_DIR@/uploads") + +message(STATUS "FileUpload:1") +file(UPLOAD + ${filename} + ${urlbase}/file1.png + TIMEOUT 2 + ) + +message(STATUS "FileUpload:2") +file(UPLOAD + ${filename} + ${urlbase}/file2.png + STATUS status + LOG log + SHOW_PROGRESS + ) + +execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum + "@CMAKE_CURRENT_BINARY_DIR@/uploads/file1.png" + OUTPUT_VARIABLE sum1 + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT sum1 MATCHES "^d16778650db435bda3a8c3435c3ff5d1 .*/uploads/file1.png$") + message(FATAL_ERROR "file1.png did not upload correctly (sum1='${sum1}')") +endif() + +execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum + "@CMAKE_CURRENT_BINARY_DIR@/uploads/file2.png" + OUTPUT_VARIABLE sum2 + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT sum2 MATCHES "^d16778650db435bda3a8c3435c3ff5d1 .*/uploads/file2.png$") + message(FATAL_ERROR "file2.png did not upload correctly (sum2='${sum2}')") +endif() + +message(STATUS "log='${log}'") +message(STATUS "status='${status}'") +message(STATUS "DONE") From 61a83f955a613383b78cb2759d867d3133741a49 Mon Sep 17 00:00:00 2001 From: David Cole Date: Mon, 28 Feb 2011 10:47:23 -0500 Subject: [PATCH 2/2] Fix KWStyle line too long error (#11286) --- Source/cmFileCommand.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 8849a470c..a72feb1e7 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -3045,7 +3045,8 @@ cmFileCommand::HandleUploadCommand(std::vector const& args) check_curl_result(res, "UPLOAD cannot set input file: "); // and give the size of the upload (optional) - res = ::curl_easy_setopt(curl, CURLOPT_INFILESIZE, static_cast(st.st_size)); + res = ::curl_easy_setopt(curl, + CURLOPT_INFILESIZE, static_cast(st.st_size)); check_curl_result(res, "UPLOAD cannot set input file size: "); res = ::curl_easy_perform(curl);