diff --git a/Help/command/ctest_submit.rst b/Help/command/ctest_submit.rst index d9b0b7849..2b83ed967 100644 --- a/Help/command/ctest_submit.rst +++ b/Help/command/ctest_submit.rst @@ -37,3 +37,16 @@ timed-out submission before attempting to re-submit. The RETRY_COUNT option specifies how many times to retry a timed-out submission. + +Submit to CDash Upload API +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + ctest_submit(CDASH_UPLOAD [CDASH_UPLOAD_TYPE ]) + +This second signature is used to upload files to CDash via the CDash +file upload API. The api first sends a request to upload to CDash along +with a content hash of the file. If CDash does not already have the file, +then it is uploaded. Along with the file, a CDash type string is specified +to tell CDash which handler to use to process the data. diff --git a/Help/manual/cmake-modules.7.rst b/Help/manual/cmake-modules.7.rst index 0a0ca2361..db5601098 100644 --- a/Help/manual/cmake-modules.7.rst +++ b/Help/manual/cmake-modules.7.rst @@ -63,6 +63,7 @@ All Modules /module/CPack /module/CPackWIX /module/CTest + /module/CTestCoverageCollectGCOV /module/CTestScriptMode /module/CTestUseLaunchers /module/Dart diff --git a/Help/module/CTestCoverageCollectGCOV.rst b/Help/module/CTestCoverageCollectGCOV.rst new file mode 100644 index 000000000..4c5deca4d --- /dev/null +++ b/Help/module/CTestCoverageCollectGCOV.rst @@ -0,0 +1 @@ +.. cmake-module:: ../../Modules/CTestCoverageCollectGCOV.cmake diff --git a/Modules/CTestCoverageCollectGCOV.cmake b/Modules/CTestCoverageCollectGCOV.cmake new file mode 100644 index 000000000..f6616e0df --- /dev/null +++ b/Modules/CTestCoverageCollectGCOV.cmake @@ -0,0 +1,138 @@ +#.rst: +# CTestCoverageCollectGCOV +# ------------------------ +# +# This module provides the function ``ctest_coverage_collect_gcov``. +# The function will run gcov on the .gcda files in a binary tree and then +# package all of the .gcov files into a tar file with a data.json that +# contains the source and build directories for CDash to use in parsing +# the coverage data. In addtion the Labels.json files for targets that +# have coverage information are also put in the tar file for CDash to +# asign the correct labels. This file can be sent to a CDash server for +# display with the +# :command:`ctest_submit(CDASH_UPLOAD)` command. +# +# .. command:: cdash_coverage_collect_gcov +# +# :: +# +# ctest_coverage_collect_gcov(TARBALL +# [SOURCE ][BUILD ] +# [GCOV_COMMAND ] +# ) +# +# Run gcov and package a tar file for CDash. The options are: +# +# ``TARBALL `` +# Specify the location of the ``.tar`` file to be created for later +# upload to CDash. Relative paths will be interpreted with respect +# to the top-level build directory. +# +# ``SOURCE `` +# Specify the top-level source directory for the build. +# Default is the value of :variable:`CTEST_SOURCE_DIRECTORY`. +# +# ``BUILD `` +# Specify the top-level build directory for the build. +# Default is the value of :variable:`CTEST_BINARY_DIRECTORY`. +# +# ``GCOV_COMMAND `` +# Specify the full path to the ``gcov`` command on the machine. +# Default is the value of :variable:`CTEST_COVERAGE_COMMAND`. + +#============================================================================= +# Copyright 2014-2015 Kitware, Inc. +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) +include(CMakeParseArguments) +function(ctest_coverage_collect_gcov) + set(options "") + set(oneValueArgs TARBALL SOURCE BUILD GCOV_COMMAND) + set(multiValueArgs "") + cmake_parse_arguments(GCOV "${options}" "${oneValueArgs}" + "${multiValueArgs}" "" ${ARGN} ) + if(NOT DEFINED GCOV_TARBALL) + message(FATAL_ERROR + "TARBALL must be specified. for ctest_coverage_collect_gcov") + endif() + if(NOT DEFINED GCOV_SOURCE) + set(source_dir "${CTEST_SOURCE_DIRECTORY}") + else() + set(source_dir "${GCOV_SOURCE}") + endif() + if(NOT DEFINED GCOV_BUILD) + set(binary_dir "${CTEST_BINARY_DIRECTORY}") + else() + set(binary_dir "${GCOV_BUILD}") + endif() + if(NOT DEFINED GCOV_GCOV_COMMAND) + set(gcov_command "${CTEST_COVERAGE_COMMAND}") + else() + set(gcov_command "${GCOV_GCOV_COMMAND}") + endif() + # run gcov on each gcda file in the binary tree + set(gcda_files) + set(label_files) + # look for gcda files in the target directories + # could do a glob from the top of the binary tree but + # this will be faster and only look where the files will be + file(STRINGS "${binary_dir}/CMakeFiles/TargetDirectories.txt" target_dirs) + foreach(target_dir ${target_dirs}) + file(GLOB_RECURSE gfiles RELATIVE ${binary_dir} "${target_dir}/*.gcda") + list(LENGTH gfiles len) + # if we have gcda files then also grab the labels file for that target + if(${len} GREATER 0) + file(GLOB_RECURSE lfiles RELATIVE ${binary_dir} + "${target_dir}/Labels.json") + list(APPEND gcda_files ${gfiles}) + list(APPEND label_files ${lfiles}) + endif() + endforeach() + # return early if no coverage files were found + list(LENGTH gcda_files len) + if(len EQUAL 0) + message("ctest_coverage_collect_gcov: No .gcda files found, " + "ignoring coverage request.") + return() + endif() + # setup the dir for the coverage files + set(coverage_dir "${binary_dir}/Testing/CoverageInfo") + file(MAKE_DIRECTORY "${coverage_dir}") + # call gcov on each .gcda file + foreach (gcda_file ${gcda_files}) + # get the directory of the gcda file + get_filename_component(gcda_file ${binary_dir}/${gcda_file} ABSOLUTE) + get_filename_component(gcov_dir ${gcda_file} DIRECTORY) + # run gcov, this will produce the .gcov file in the current + # working directory + execute_process(COMMAND + ${gcov_command} -b -o ${gcov_dir} ${gcda_file} + OUTPUT_VARIABLE out + WORKING_DIRECTORY ${coverage_dir}) + endforeach() + # create json file with project information + file(WRITE ${coverage_dir}/data.json + "{ + \"Source\": \"${source_dir}\", + \"Binary\": \"${binary_dir}\" +}") + # collect the gcov files + set(gcov_files) + file(GLOB_RECURSE gcov_files RELATIVE ${binary_dir} "${coverage_dir}/*.gcov") + # tar up the coverage info with the same date so that the md5 + # sum will be the same for the tar file independent of file time + # stamps + execute_process(COMMAND + ${CMAKE_COMMAND} -E tar cvfj ${GCOV_TARBALL} + "--mtime=1970-01-01 0:0:0 UTC" ${gcov_files} + ${coverage_dir}/data.json ${label_files} + WORKING_DIRECTORY ${binary_dir}) +endfunction() diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index af4c7cfcc..c04cf9abc 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -537,6 +537,7 @@ set(CTEST_SRCS cmCTest.cxx CTest/cmCTestConfigureHandler.cxx CTest/cmCTestCoverageCommand.cxx CTest/cmCTestCoverageHandler.cxx + CTest/cmCTestCurl.cxx CTest/cmParseMumpsCoverage.cxx CTest/cmParseCacheCoverage.cxx CTest/cmParseGTMCoverage.cxx diff --git a/Source/CTest/cmCTestCurl.cxx b/Source/CTest/cmCTestCurl.cxx new file mode 100644 index 000000000..b0d26cc80 --- /dev/null +++ b/Source/CTest/cmCTestCurl.cxx @@ -0,0 +1,271 @@ +/*============================================================================ + CMake - Cross Platform Makefile Generator + Copyright 2000-2015 Kitware, Inc. + + Distributed under the OSI-approved BSD License (the "License"); + see accompanying file Copyright.txt for details. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the License for more information. +============================================================================*/ +#include "cmCTestCurl.h" + +#include "cmSystemTools.h" +#include "cmCTest.h" + +cmCTestCurl::cmCTestCurl(cmCTest* ctest) +{ + this->CTest = ctest; + this->SetProxyType(); + this->UseHttp10 = false; + // In windows, this will init the winsock stuff + ::curl_global_init(CURL_GLOBAL_ALL); + // default is to verify https + this->VerifyPeerOff = false; + this->VerifyHostOff = false; + this->TimeOutSeconds = 0; +} + +namespace +{ +static size_t +curlWriteMemoryCallback(void *ptr, size_t size, size_t nmemb, + void *data) +{ + int realsize = (int)(size * nmemb); + + std::vector *vec + = static_cast* >(data); + const char* chPtr = static_cast(ptr); + vec->insert(vec->end(), chPtr, chPtr + realsize); + return realsize; +} + +static size_t +curlDebugCallback(CURL *, curl_infotype, char *chPtr, + size_t size, void *data) +{ + std::vector *vec + = static_cast* >(data); + vec->insert(vec->end(), chPtr, chPtr + size); + + return size; +} + +} + +void cmCTestCurl::SetCurlOptions(std::vector const& args) +{ + for( std::vector::const_iterator i = args.begin(); + i != args.end(); ++i) + { + if(*i == "CURLOPT_SSL_VERIFYPEER_OFF") + { + this->VerifyPeerOff = true; + } + if(*i == "CURLOPT_SSL_VERIFYHOST_OFF") + { + this->VerifyHostOff = true; + } + } +} + +bool cmCTestCurl::InitCurl() +{ + this->Curl = curl_easy_init(); + if(!this->Curl) + { + return false; + } + if(this->VerifyPeerOff) + { + curl_easy_setopt(this->Curl, CURLOPT_SSL_VERIFYPEER, 0); + } + if(this->VerifyHostOff) + { + curl_easy_setopt(this->Curl, CURLOPT_SSL_VERIFYHOST, 0); + } + if(this->HTTPProxy.size()) + { + curl_easy_setopt(this->Curl, CURLOPT_PROXY, this->HTTPProxy.c_str()); + curl_easy_setopt(this->Curl, CURLOPT_PROXYTYPE, this->HTTPProxyType); + if (this->HTTPProxyAuth.size() > 0) + { + curl_easy_setopt(this->Curl, CURLOPT_PROXYUSERPWD, + this->HTTPProxyAuth.c_str()); + } + } + if(this->UseHttp10) + { + curl_easy_setopt(this->Curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + } + // enable HTTP ERROR parsing + curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1); + return true; +} + + +bool cmCTestCurl::UploadFile(std::string const& local_file, + std::string const& url, + std::string const& fields, + std::string& response) +{ + response = ""; + if(!this->InitCurl()) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Initialization of curl failed"); + return false; + } + /* enable uploading */ + curl_easy_setopt(this->Curl, CURLOPT_UPLOAD, 1); + // if there is little to no activity for too long stop submitting + if(this->TimeOutSeconds) + { + ::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_LIMIT, 1); + ::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_TIME, + this->TimeOutSeconds); + } + /* HTTP PUT please */ + ::curl_easy_setopt(this->Curl, CURLOPT_PUT, 1); + ::curl_easy_setopt(this->Curl, CURLOPT_VERBOSE, 1); + + FILE* ftpfile = cmsys::SystemTools::Fopen(local_file, "rb"); + if(!ftpfile) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Could not open file for upload: " << local_file << "\n"); + return false; + } + // set the url + std::string upload_url = url; + upload_url += "?"; + upload_url += fields; + ::curl_easy_setopt(this->Curl, CURLOPT_URL, upload_url.c_str()); + // now specify which file to upload + ::curl_easy_setopt(this->Curl, CURLOPT_INFILE, ftpfile); + unsigned long filelen = cmSystemTools::FileLength(local_file); + // and give the size of the upload (optional) + ::curl_easy_setopt(this->Curl, CURLOPT_INFILESIZE, + static_cast(filelen)); + ::curl_easy_setopt(this->Curl, CURLOPT_WRITEFUNCTION, + curlWriteMemoryCallback); + ::curl_easy_setopt(this->Curl, CURLOPT_DEBUGFUNCTION, + curlDebugCallback); + std::vector responseData; + std::vector debugData; + ::curl_easy_setopt(this->Curl, CURLOPT_FILE, (void *)&responseData); + ::curl_easy_setopt(this->Curl, CURLOPT_DEBUGDATA, (void *)&debugData); + ::curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1); + // Now run off and do what you've been told! + ::curl_easy_perform(this->Curl); + ::fclose(ftpfile); + ::curl_global_cleanup(); + + if ( responseData.size() > 0 ) + { + response = std::string(responseData.begin(), responseData.end()); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Curl response: [" << response << "]\n"); + } + std::string curlDebug; + if ( debugData.size() > 0 ) + { + curlDebug = std::string(debugData.begin(), debugData.end()); + cmCTestLog(this->CTest, DEBUG, "Curl debug: [" << curlDebug << "]\n"); + } + if(response.size() == 0) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "No response from server.\n" << + curlDebug); + return false; + } + return true; +} + +bool cmCTestCurl::HttpRequest(std::string const& url, + std::string const& fields, + std::string& response) +{ + response = ""; + cmCTestLog(this->CTest, DEBUG, "HttpRequest\n" + << "url: " << url << "\n" + << "fields " << fields << "\n"); + if(!this->InitCurl()) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Initialization of curl failed"); + return false; + } + curl_easy_setopt(this->Curl, CURLOPT_POST, 1); + curl_easy_setopt(this->Curl, CURLOPT_POSTFIELDS, fields.c_str()); + ::curl_easy_setopt(this->Curl, CURLOPT_URL, url.c_str()); + ::curl_easy_setopt(this->Curl, CURLOPT_FOLLOWLOCATION, 1); + //set response options + ::curl_easy_setopt(this->Curl, CURLOPT_WRITEFUNCTION, + curlWriteMemoryCallback); + ::curl_easy_setopt(this->Curl, CURLOPT_DEBUGFUNCTION, + curlDebugCallback); + std::vector responseData; + std::vector debugData; + ::curl_easy_setopt(this->Curl, CURLOPT_FILE, (void *)&responseData); + ::curl_easy_setopt(this->Curl, CURLOPT_DEBUGDATA, (void *)&debugData); + ::curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1); + + CURLcode res = ::curl_easy_perform(this->Curl); + + ::curl_easy_cleanup(this->Curl); + ::curl_global_cleanup(); + if ( responseData.size() > 0 ) + { + response = std::string(responseData.begin(), responseData.end()); + cmCTestLog(this->CTest, DEBUG, "Curl response: [" << response << "]\n"); + } + if ( debugData.size() > 0 ) + { + std::string curlDebug = std::string(debugData.begin(), debugData.end()); + cmCTestLog(this->CTest, DEBUG, "Curl debug: [" << curlDebug << "]\n"); + } + cmCTestLog(this->CTest, DEBUG, "Curl res: " << res << "\n"); + return (res == 0); +} + +void cmCTestCurl::SetProxyType() +{ + if ( cmSystemTools::GetEnv("HTTP_PROXY") ) + { + this->HTTPProxy = cmSystemTools::GetEnv("HTTP_PROXY"); + if ( cmSystemTools::GetEnv("HTTP_PROXY_PORT") ) + { + this->HTTPProxy += ":"; + this->HTTPProxy += cmSystemTools::GetEnv("HTTP_PROXY_PORT"); + } + if ( cmSystemTools::GetEnv("HTTP_PROXY_TYPE") ) + { + // this is the default + this->HTTPProxyType = CURLPROXY_HTTP; + std::string type = cmSystemTools::GetEnv("HTTP_PROXY_TYPE"); + // HTTP/SOCKS4/SOCKS5 + if ( type == "HTTP" ) + { + this->HTTPProxyType = CURLPROXY_HTTP; + } + else if ( type == "SOCKS4" ) + { + this->HTTPProxyType = CURLPROXY_SOCKS4; + } + else if ( type == "SOCKS5" ) + { + this->HTTPProxyType = CURLPROXY_SOCKS5; + } + } + if ( cmSystemTools::GetEnv("HTTP_PROXY_USER") ) + { + this->HTTPProxyAuth = cmSystemTools::GetEnv("HTTP_PROXY_USER"); + } + if ( cmSystemTools::GetEnv("HTTP_PROXY_PASSWD") ) + { + this->HTTPProxyAuth += ":"; + this->HTTPProxyAuth += cmSystemTools::GetEnv("HTTP_PROXY_PASSWD"); + } + } +} diff --git a/Source/CTest/cmCTestCurl.h b/Source/CTest/cmCTestCurl.h new file mode 100644 index 000000000..5bb8b4111 --- /dev/null +++ b/Source/CTest/cmCTestCurl.h @@ -0,0 +1,52 @@ +/*============================================================================ + CMake - Cross Platform Makefile Generator + Copyright 2000-2015 Kitware, Inc. + + Distributed under the OSI-approved BSD License (the "License"); + see accompanying file Copyright.txt for details. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the License for more information. +============================================================================*/ +#ifndef cmCTestCurl_h +#define cmCTestCurl_h + +#include "cmStandardIncludes.h" + +#include "cm_curl.h" + +class cmCTest; + +class cmCTestCurl +{ +public: + cmCTestCurl(cmCTest*); + bool UploadFile(std::string const& url, + std::string const& file, + std::string const& fields, + std::string& response); + bool HttpRequest(std::string const& url, + std::string const& fields, + std::string& response); + // currently only supports CURLOPT_SSL_VERIFYPEER_OFF + // and CURLOPT_SSL_VERIFYHOST_OFF + void SetCurlOptions(std::vector const& args); + void SetUseHttp10On() { this->UseHttp10 = true;} + void SetTimeOutSeconds(int s) { this->TimeOutSeconds = s;} +protected: + void SetProxyType(); + bool InitCurl(); +private: + cmCTest* CTest; + CURL* Curl; + std::string HTTPProxyAuth; + std::string HTTPProxy; + curl_proxytype HTTPProxyType; + bool VerifyHostOff; + bool VerifyPeerOff; + bool UseHttp10; + int TimeOutSeconds; +}; + +#endif diff --git a/Source/CTest/cmCTestSubmitCommand.cxx b/Source/CTest/cmCTestSubmitCommand.cxx index 4005a6348..cc3514ff3 100644 --- a/Source/CTest/cmCTestSubmitCommand.cxx +++ b/Source/CTest/cmCTestSubmitCommand.cxx @@ -27,7 +27,8 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() = this->Makefile->GetDefinition("CTEST_TRIGGER_SITE"); bool ctestDropSiteCDash = this->Makefile->IsOn("CTEST_DROP_SITE_CDASH"); - + const char* ctestProjectName + = this->Makefile->GetDefinition("CTEST_PROJECT_NAME"); if ( !ctestDropMethod ) { ctestDropMethod = "http"; @@ -43,7 +44,7 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() // error: CDash requires CTEST_DROP_LOCATION definition // in CTestConfig.cmake } - + this->CTest->SetCTestConfiguration("ProjectName", ctestProjectName); this->CTest->SetCTestConfiguration("DropMethod", ctestDropMethod); this->CTest->SetCTestConfiguration("DropSite", ctestDropSite); this->CTest->SetCTestConfiguration("DropLocation", ctestDropLocation); @@ -144,44 +145,75 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() static_cast(handler)->SetOption("InternalTest", this->InternalTest ? "ON" : "OFF"); + if (this->CDashUpload) + { + static_cast(handler)-> + SetOption("CDashUploadFile", this->CDashUploadFile.c_str()); + static_cast(handler)-> + SetOption("CDashUploadType", this->CDashUploadType.c_str()); + } return handler; } +//---------------------------------------------------------------------------- +bool cmCTestSubmitCommand::InitialPass(std::vector const& args, + cmExecutionStatus& status) +{ + this->CDashUpload = !args.empty() && args[0] == "CDASH_UPLOAD"; + return this->cmCTestHandlerCommand::InitialPass(args, status); +} //---------------------------------------------------------------------------- bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg) { - // Look for arguments specific to this command. - if(arg == "PARTS") + if (this->CDashUpload) { - this->ArgumentDoing = ArgumentDoingParts; - this->PartsMentioned = true; - return true; - } + if(arg == "CDASH_UPLOAD") + { + this->ArgumentDoing = ArgumentDoingCDashUpload; + return true; + } - if(arg == "FILES") - { - this->ArgumentDoing = ArgumentDoingFiles; - this->FilesMentioned = true; - return true; + if(arg == "CDASH_UPLOAD_TYPE") + { + this->ArgumentDoing = ArgumentDoingCDashUploadType; + return true; + } } - - if(arg == "RETRY_COUNT") + else { - this->ArgumentDoing = ArgumentDoingRetryCount; - return true; - } + // Look for arguments specific to this command. + if(arg == "PARTS") + { + this->ArgumentDoing = ArgumentDoingParts; + this->PartsMentioned = true; + return true; + } - if(arg == "RETRY_DELAY") - { - this->ArgumentDoing = ArgumentDoingRetryDelay; - return true; - } + if(arg == "FILES") + { + this->ArgumentDoing = ArgumentDoingFiles; + this->FilesMentioned = true; + return true; + } - if(arg == "INTERNAL_TEST_CHECKSUM") - { - this->InternalTest = true; - return true; + if(arg == "RETRY_COUNT") + { + this->ArgumentDoing = ArgumentDoingRetryCount; + return true; + } + + if(arg == "RETRY_DELAY") + { + this->ArgumentDoing = ArgumentDoingRetryDelay; + return true; + } + + if(arg == "INTERNAL_TEST_CHECKSUM") + { + this->InternalTest = true; + return true; + } } // Look for other arguments. @@ -240,6 +272,20 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg) return true; } + if(this->ArgumentDoing == ArgumentDoingCDashUpload) + { + this->ArgumentDoing = ArgumentDoingNone; + this->CDashUploadFile = arg; + return true; + } + + if(this->ArgumentDoing == ArgumentDoingCDashUploadType) + { + this->ArgumentDoing = ArgumentDoingNone; + this->CDashUploadType = arg; + return true; + } + // Look for other arguments. return this->Superclass::CheckArgumentValue(arg); } diff --git a/Source/CTest/cmCTestSubmitCommand.h b/Source/CTest/cmCTestSubmitCommand.h index 3673fbdec..19e8eafb5 100644 --- a/Source/CTest/cmCTestSubmitCommand.h +++ b/Source/CTest/cmCTestSubmitCommand.h @@ -32,6 +32,7 @@ public: this->InternalTest = false; this->RetryCount = ""; this->RetryDelay = ""; + this->CDashUpload = false; } /** @@ -45,6 +46,9 @@ public: return ni; } + virtual bool InitialPass(std::vector const& args, + cmExecutionStatus &status); + /** * The name of the command as specified in CMakeList.txt. */ @@ -64,6 +68,8 @@ protected: ArgumentDoingFiles, ArgumentDoingRetryDelay, ArgumentDoingRetryCount, + ArgumentDoingCDashUpload, + ArgumentDoingCDashUploadType, ArgumentDoingLast2 }; @@ -74,6 +80,9 @@ protected: cmCTest::SetOfStrings Files; std::string RetryCount; std::string RetryDelay; + bool CDashUpload; + std::string CDashUploadFile; + std::string CDashUploadType; }; diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index bc6fb3138..11e334385 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -10,7 +10,8 @@ See the License for more information. ============================================================================*/ #include "cmCTestSubmitHandler.h" - +#include "cmCTestScriptHandler.h" +#include "cmake.h" #include "cmSystemTools.h" #include "cmVersion.h" #include "cmGeneratedFileStream.h" @@ -23,8 +24,10 @@ // For XML-RPC submission #include "cm_xmlrpc.h" +#include // For curl submission #include "cm_curl.h" +#include "cmCTestCurl.h" #include @@ -1055,9 +1058,171 @@ bool cmCTestSubmitHandler::SubmitUsingXMLRPC(std::string const&, } #endif +void cmCTestSubmitHandler::ConstructCDashURL(std::string& dropMethod, + std::string& url) +{ + dropMethod = this->CTest->GetCTestConfiguration("DropMethod"); + url = dropMethod; + url += "://"; + if ( this->CTest->GetCTestConfiguration("DropSiteUser").size() > 0 ) + { + url += this->CTest->GetCTestConfiguration("DropSiteUser"); + cmCTestLog(this->CTest, HANDLER_OUTPUT, + this->CTest->GetCTestConfiguration("DropSiteUser").c_str()); + if ( this->CTest->GetCTestConfiguration("DropSitePassword").size() > 0 ) + { + url += ":" + this->CTest->GetCTestConfiguration("DropSitePassword"); + cmCTestLog(this->CTest, HANDLER_OUTPUT, ":******"); + } + url += "@"; + } + url += this->CTest->GetCTestConfiguration("DropSite") + + this->CTest->GetCTestConfiguration("DropLocation"); +} + + +int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file, + std::string const& typeString) +{ + if (file.empty()) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Upload file not specified\n"); + return -1; + } + if (!cmSystemTools::FileExists(file)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Upload file not found: '" << file << "'\n"); + return -1; + } + cmCTestCurl curl(this->CTest); + std::string curlopt(this->CTest->GetCTestConfiguration("CurlOptions")); + std::vector args; + cmSystemTools::ExpandListArgument(curlopt, args); + curl.SetCurlOptions(args); + curl.SetTimeOutSeconds(SUBMIT_TIMEOUT_IN_SECONDS_DEFAULT); + std::string dropMethod; + std::string url; + this->ConstructCDashURL(dropMethod, url); + std::string::size_type pos = url.find("submit.php?"); + url = url.substr(0, pos+10); + if ( ! (dropMethod == "http" || dropMethod == "https" ) ) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Only http and https are supported for CDASH_UPLOAD\n"); + return -1; + } + char md5sum[33]; + md5sum[32] = 0; + cmSystemTools::ComputeFileMD5(file, md5sum); + // 1. request the buildid and check to see if the file + // has already been uploaded + // TODO I added support for subproject. You would need to add + // a "&subproject=subprojectname" to the first POST. + cmCTestScriptHandler* ch = + static_cast(this->CTest->GetHandler("script")); + cmake* cm = ch->GetCMake(); + const char* subproject = cm->GetProperty("SubProject", cmProperty::GLOBAL); + // TODO: Encode values for a URL instead of trusting caller. + std::ostringstream str; + str << "project=" + << this->CTest->GetCTestConfiguration("ProjectName") << "&"; + if(subproject) + { + str << "subproject=" << subproject << "&"; + } + str << "stamp=" << this->CTest->GetCurrentTag() << "-" + << this->CTest->GetTestModelString() << "&" + << "model=" << this->CTest->GetTestModelString() << "&" + << "build=" << this->CTest->GetCTestConfiguration("BuildName") << "&" + << "site=" << this->CTest->GetCTestConfiguration("Site") << "&" + << "track=" << this->CTest->GetTestModelString() << "&" + << "starttime=" << (int)cmSystemTools::GetTime() << "&" + << "endtime=" << (int)cmSystemTools::GetTime() << "&" + << "datafilesmd5[0]=" << md5sum << "&" + << "type=" << typeString; + std::string fields = str.str(); + cmCTestLog(this->CTest, DEBUG, "fields: " << fields << "\nurl:" + << url << "\nfile: " << file << "\n"); + std::string response; + if(!curl.HttpRequest(url, fields, response)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error in HttpRequest\n" << response); + return -1; + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Request upload response: [" << response << "]\n"); + Json::Value json; + Json::Reader reader; + if(!reader.parse(response, json)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "error parsing json string [" << response << "]\n" + << reader.getFormattedErrorMessages() << "\n"); + return -1; + } + if(json["status"].asInt() != 0) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Bad status returned from CDash: " + << json["status"].asInt()); + return -1; + } + if(json["datafilesmd5"].isArray()) + { + int datares = json["datafilesmd5"][0].asInt(); + if(datares == 1) + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "File already exists on CDash, skip upload " + << file << "\n"); + return 0; + } + } + else + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "bad datafilesmd5 value in response " + << response << "\n"); + return -1; + } + + std::string upload_as = cmSystemTools::GetFilenameName(file); + std::ostringstream fstr; + fstr << "type=" << typeString << "&" + << "md5=" << md5sum << "&" + << "filename=" << upload_as << "&" + << "buildid=" << json["buildid"].asString(); + if(!curl.UploadFile(file, url, fstr.str(), response)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "error uploading to CDash. " + << file << " " << url << " " << fstr.str()); + return -1; + } + if(!reader.parse(response, json)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "error parsing json string [" << response << "]\n" + << reader.getFormattedErrorMessages() << "\n"); + return -1; + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Upload file response: [" << response << "]\n"); + return 0; +} + //---------------------------------------------------------------------------- int cmCTestSubmitHandler::ProcessHandler() { + const char* cdashUploadFile = this->GetOption("CDashUploadFile"); + const char* cdashUploadType = this->GetOption("CDashUploadType"); + if(cdashUploadFile && cdashUploadType) + { + return this->HandleCDashUploadFile(cdashUploadFile, cdashUploadType); + } std::string iscdash = this->CTest->GetCTestConfiguration("IsCDash"); // cdash does not need to trigger so just return true if(!iscdash.empty()) diff --git a/Source/CTest/cmCTestSubmitHandler.h b/Source/CTest/cmCTestSubmitHandler.h index accabd1c5..f9cd8946e 100644 --- a/Source/CTest/cmCTestSubmitHandler.h +++ b/Source/CTest/cmCTestSubmitHandler.h @@ -41,6 +41,11 @@ public: /** Specify a set of files to submit. */ void SelectFiles(cmCTest::SetOfStrings const& files); + // handle the cdash file upload protocol + int HandleCDashUploadFile(std::string const& file, std::string const& type); + + void ConstructCDashURL(std::string& dropMethod, std::string& url); + private: void SetLogFile(std::ostream* ost) { this->LogFile = ost; } diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadFILES-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadFILES-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadFILES-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadFILES-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadFILES-stderr.txt new file mode 100644 index 000000000..48177e2c0 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadFILES-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/CDashUploadFILES/test.cmake:[0-9]+ \(ctest_submit\): + ctest_submit called with unknown argument "FILES". diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadFTP-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadFTP-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadFTP-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadFTP-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadFTP-stderr.txt new file mode 100644 index 000000000..77df44fec --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadFTP-stderr.txt @@ -0,0 +1 @@ +Only http and https are supported for CDASH_UPLOAD diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadNone-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadNone-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadNone-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadNone-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadNone-stderr.txt new file mode 100644 index 000000000..af95b5cf5 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadNone-stderr.txt @@ -0,0 +1 @@ +Upload file not specified diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-stderr.txt new file mode 100644 index 000000000..497ead298 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadPARTS-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/CDashUploadPARTS/test.cmake:[0-9]+ \(ctest_submit\): + ctest_submit called with unknown argument "PARTS". diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-stderr.txt new file mode 100644 index 000000000..8c4e6b136 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_COUNT/test.cmake:[0-9]+ \(ctest_submit\): + ctest_submit called with unknown argument "RETRY_COUNT". diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-result.txt b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-stderr.txt b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-stderr.txt new file mode 100644 index 000000000..6c56399d9 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/CDashUploadRETRY_DELAY/test.cmake:[0-9]+ \(ctest_submit\): + ctest_submit called with unknown argument "RETRY_DELAY". diff --git a/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-result.txt b/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-stderr.txt b/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-stderr.txt new file mode 100644 index 000000000..dfa7e33fa --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/PARTSCDashUpload-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/PARTSCDashUpload/test.cmake:[0-9]+ \(ctest_submit\): + Part name "CDASH_UPLOAD" is invalid. diff --git a/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-result.txt b/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-result.txt new file mode 100644 index 000000000..b57e2deb7 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-stderr.txt b/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-stderr.txt new file mode 100644 index 000000000..42becafd9 --- /dev/null +++ b/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType-stderr.txt @@ -0,0 +1,2 @@ +CMake Error at .*/Tests/RunCMake/CTestSubmit/PARTSCDashUploadType/test.cmake:[0-9]+ \(ctest_submit\): + Part name "CDASH_UPLOAD_TYPE" is invalid. diff --git a/Tests/RunCMake/CTestSubmit/RunCMakeTest.cmake b/Tests/RunCMake/CTestSubmit/RunCMakeTest.cmake index 3638007e4..797365d71 100644 --- a/Tests/RunCMake/CTestSubmit/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestSubmit/RunCMakeTest.cmake @@ -33,6 +33,19 @@ run_ctest_submit(BadArg bad-arg) run_ctest_submit(BadPARTS PARTS bad-part) run_ctest_submit(BadFILES FILES bad-file) run_ctest_submit(RepeatRETURN_VALUE RETURN_VALUE res RETURN_VALUE res) +run_ctest_submit(PARTSCDashUpload PARTS Configure CDASH_UPLOAD) +run_ctest_submit(PARTSCDashUploadType PARTS Configure CDASH_UPLOAD_TYPE) +run_ctest_submit(CDashUploadPARTS CDASH_UPLOAD bad-upload PARTS) +run_ctest_submit(CDashUploadFILES CDASH_UPLOAD bad-upload FILES) +run_ctest_submit(CDashUploadRETRY_COUNT CDASH_UPLOAD bad-upload RETRY_COUNT) +run_ctest_submit(CDashUploadRETRY_DELAY CDASH_UPLOAD bad-upload RETRY_DELAY) +run_ctest_submit(CDashUploadNone CDASH_UPLOAD) + +function(run_ctest_CDashUploadFTP) + set(CASE_DROP_METHOD ftp) + run_ctest_submit(CDashUploadFTP CDASH_UPLOAD ${CMAKE_CURRENT_LIST_FILE}) +endfunction() +run_ctest_CDashUploadFTP() #----------------------------------------------------------------------------- # Test failed drops by various protocols