ctest_submit: Add CDASH_UPLOAD mode to upload files to CDash

This adds support for the new cdash API where arbitrary files can be
uploaded to the CDash server.  This CDash API communicates via json
files so the json parser jsoncpp was added to the Utilities directory.
This commit is contained in:
Bill Hoffman 2015-01-14 10:54:25 -05:00 committed by Brad King
parent 17cfa09eb2
commit 5dc33f89b5
8 changed files with 535 additions and 3 deletions

View File

@ -37,3 +37,14 @@ timed-out submission before attempting to re-submit.
The RETRY_COUNT option specifies how many times to retry a timed-out
submission.
::
ctest_submit([CDASH_UPLOAD file]
[CDASH_UPLOAD_TYPE type_string])
This second signature is used to upload files to CDash via the CDash
file upload API. The api first sends a request to upload to CDash along
with the md5 sum of the file. If CDash does not already have the file,
then it is uploaded. Along with the file, a CDash type string is specified
to tell CDash which handler to use to process the data.

View File

@ -537,6 +537,7 @@ set(CTEST_SRCS cmCTest.cxx
CTest/cmCTestConfigureHandler.cxx
CTest/cmCTestCoverageCommand.cxx
CTest/cmCTestCoverageHandler.cxx
CTest/cmCTestCurl.cxx
CTest/cmParseMumpsCoverage.cxx
CTest/cmParseCacheCoverage.cxx
CTest/cmParseGTMCoverage.cxx

View File

@ -0,0 +1,271 @@
/*============================================================================
CMake - Cross Platform Makefile Generator
Copyright 2000-2015 Kitware, Inc.
Distributed under the OSI-approved BSD License (the "License");
see accompanying file Copyright.txt for details.
This software is distributed WITHOUT ANY WARRANTY; without even the
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the License for more information.
============================================================================*/
#include "cmCTestCurl.h"
#include "cmSystemTools.h"
#include "cmCTest.h"
cmCTestCurl::cmCTestCurl(cmCTest* ctest)
{
this->CTest = ctest;
this->SetProxyType();
this->UseHttp10 = false;
// In windows, this will init the winsock stuff
::curl_global_init(CURL_GLOBAL_ALL);
// default is to verify https
this->VerifyPeerOff = false;
this->VerifyHostOff = false;
this->TimeOutSeconds = 0;
}
namespace
{
static size_t
curlWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
void *data)
{
int realsize = (int)(size * nmemb);
std::vector<char> *vec
= static_cast<std::vector<char>* >(data);
const char* chPtr = static_cast<char*>(ptr);
vec->insert(vec->end(), chPtr, chPtr + realsize);
return realsize;
}
static size_t
curlDebugCallback(CURL *, curl_infotype, char *chPtr,
size_t size, void *data)
{
std::vector<char> *vec
= static_cast<std::vector<char>* >(data);
vec->insert(vec->end(), chPtr, chPtr + size);
return size;
}
}
void cmCTestCurl::SetCurlOptions(std::vector<std::string> const& args)
{
for( std::vector<std::string>::const_iterator i = args.begin();
i != args.end(); ++i)
{
if(*i == "CURLOPT_SSL_VERIFYPEER_OFF")
{
this->VerifyPeerOff = true;
}
if(*i == "CURLOPT_SSL_VERIFYHOST_OFF")
{
this->VerifyHostOff = true;
}
}
}
bool cmCTestCurl::InitCurl()
{
this->Curl = curl_easy_init();
if(!this->Curl)
{
return false;
}
if(this->VerifyPeerOff)
{
curl_easy_setopt(this->Curl, CURLOPT_SSL_VERIFYPEER, 0);
}
if(this->VerifyHostOff)
{
curl_easy_setopt(this->Curl, CURLOPT_SSL_VERIFYHOST, 0);
}
if(this->HTTPProxy.size())
{
curl_easy_setopt(this->Curl, CURLOPT_PROXY, this->HTTPProxy.c_str());
curl_easy_setopt(this->Curl, CURLOPT_PROXYTYPE, this->HTTPProxyType);
if (this->HTTPProxyAuth.size() > 0)
{
curl_easy_setopt(this->Curl, CURLOPT_PROXYUSERPWD,
this->HTTPProxyAuth.c_str());
}
}
if(this->UseHttp10)
{
curl_easy_setopt(this->Curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
}
// enable HTTP ERROR parsing
curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1);
return true;
}
bool cmCTestCurl::UploadFile(std::string const& local_file,
std::string const& url,
std::string const& fields,
std::string& response)
{
response = "";
if(!this->InitCurl())
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Initialization of curl failed");
return false;
}
/* enable uploading */
curl_easy_setopt(this->Curl, CURLOPT_UPLOAD, 1);
// if there is little to no activity for too long stop submitting
if(this->TimeOutSeconds)
{
::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_LIMIT, 1);
::curl_easy_setopt(this->Curl, CURLOPT_LOW_SPEED_TIME,
this->TimeOutSeconds);
}
/* HTTP PUT please */
::curl_easy_setopt(this->Curl, CURLOPT_PUT, 1);
::curl_easy_setopt(this->Curl, CURLOPT_VERBOSE, 1);
FILE* ftpfile = cmsys::SystemTools::Fopen(local_file, "rb");
if(!ftpfile)
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Could not open file for upload: " << local_file << "\n");
return false;
}
// set the url
std::string upload_url = url;
upload_url += "?";
upload_url += fields;
::curl_easy_setopt(this->Curl, CURLOPT_URL, upload_url.c_str());
// now specify which file to upload
::curl_easy_setopt(this->Curl, CURLOPT_INFILE, ftpfile);
unsigned long filelen = cmSystemTools::FileLength(local_file);
// and give the size of the upload (optional)
::curl_easy_setopt(this->Curl, CURLOPT_INFILESIZE,
static_cast<long>(filelen));
::curl_easy_setopt(this->Curl, CURLOPT_WRITEFUNCTION,
curlWriteMemoryCallback);
::curl_easy_setopt(this->Curl, CURLOPT_DEBUGFUNCTION,
curlDebugCallback);
std::vector<char> responseData;
std::vector<char> debugData;
::curl_easy_setopt(this->Curl, CURLOPT_FILE, (void *)&responseData);
::curl_easy_setopt(this->Curl, CURLOPT_DEBUGDATA, (void *)&debugData);
::curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1);
// Now run off and do what you've been told!
::curl_easy_perform(this->Curl);
::fclose(ftpfile);
::curl_global_cleanup();
if ( responseData.size() > 0 )
{
response = std::string(responseData.begin(), responseData.end());
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"Curl response: [" << response << "]\n");
}
std::string curlDebug;
if ( debugData.size() > 0 )
{
curlDebug = std::string(debugData.begin(), debugData.end());
cmCTestLog(this->CTest, DEBUG, "Curl debug: [" << curlDebug << "]\n");
}
if(response.size() == 0)
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "No response from server.\n" <<
curlDebug);
return false;
}
return true;
}
bool cmCTestCurl::HttpRequest(std::string const& url,
std::string const& fields,
std::string& response)
{
response = "";
cmCTestLog(this->CTest, DEBUG, "HttpRequest\n"
<< "url: " << url << "\n"
<< "fields " << fields << "\n");
if(!this->InitCurl())
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Initialization of curl failed");
return false;
}
curl_easy_setopt(this->Curl, CURLOPT_POST, 1);
curl_easy_setopt(this->Curl, CURLOPT_POSTFIELDS, fields.c_str());
::curl_easy_setopt(this->Curl, CURLOPT_URL, url.c_str());
::curl_easy_setopt(this->Curl, CURLOPT_FOLLOWLOCATION, 1);
//set response options
::curl_easy_setopt(this->Curl, CURLOPT_WRITEFUNCTION,
curlWriteMemoryCallback);
::curl_easy_setopt(this->Curl, CURLOPT_DEBUGFUNCTION,
curlDebugCallback);
std::vector<char> responseData;
std::vector<char> debugData;
::curl_easy_setopt(this->Curl, CURLOPT_FILE, (void *)&responseData);
::curl_easy_setopt(this->Curl, CURLOPT_DEBUGDATA, (void *)&debugData);
::curl_easy_setopt(this->Curl, CURLOPT_FAILONERROR, 1);
CURLcode res = ::curl_easy_perform(this->Curl);
::curl_easy_cleanup(this->Curl);
::curl_global_cleanup();
if ( responseData.size() > 0 )
{
response = std::string(responseData.begin(), responseData.end());
cmCTestLog(this->CTest, DEBUG, "Curl response: [" << response << "]\n");
}
if ( debugData.size() > 0 )
{
std::string curlDebug = std::string(debugData.begin(), debugData.end());
cmCTestLog(this->CTest, DEBUG, "Curl debug: [" << curlDebug << "]\n");
}
cmCTestLog(this->CTest, DEBUG, "Curl res: " << res << "\n");
return (res == 0);
}
void cmCTestCurl::SetProxyType()
{
if ( cmSystemTools::GetEnv("HTTP_PROXY") )
{
this->HTTPProxy = cmSystemTools::GetEnv("HTTP_PROXY");
if ( cmSystemTools::GetEnv("HTTP_PROXY_PORT") )
{
this->HTTPProxy += ":";
this->HTTPProxy += cmSystemTools::GetEnv("HTTP_PROXY_PORT");
}
if ( cmSystemTools::GetEnv("HTTP_PROXY_TYPE") )
{
// this is the default
this->HTTPProxyType = CURLPROXY_HTTP;
std::string type = cmSystemTools::GetEnv("HTTP_PROXY_TYPE");
// HTTP/SOCKS4/SOCKS5
if ( type == "HTTP" )
{
this->HTTPProxyType = CURLPROXY_HTTP;
}
else if ( type == "SOCKS4" )
{
this->HTTPProxyType = CURLPROXY_SOCKS4;
}
else if ( type == "SOCKS5" )
{
this->HTTPProxyType = CURLPROXY_SOCKS5;
}
}
if ( cmSystemTools::GetEnv("HTTP_PROXY_USER") )
{
this->HTTPProxyAuth = cmSystemTools::GetEnv("HTTP_PROXY_USER");
}
if ( cmSystemTools::GetEnv("HTTP_PROXY_PASSWD") )
{
this->HTTPProxyAuth += ":";
this->HTTPProxyAuth += cmSystemTools::GetEnv("HTTP_PROXY_PASSWD");
}
}
}

View File

@ -0,0 +1,52 @@
/*============================================================================
CMake - Cross Platform Makefile Generator
Copyright 2000-2015 Kitware, Inc.
Distributed under the OSI-approved BSD License (the "License");
see accompanying file Copyright.txt for details.
This software is distributed WITHOUT ANY WARRANTY; without even the
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the License for more information.
============================================================================*/
#ifndef cmCTestCurl_h
#define cmCTestCurl_h
#include "cmStandardIncludes.h"
#include "cm_curl.h"
class cmCTest;
class cmCTestCurl
{
public:
cmCTestCurl(cmCTest*);
bool UploadFile(std::string const& url,
std::string const& file,
std::string const& fields,
std::string& response);
bool HttpRequest(std::string const& url,
std::string const& fields,
std::string& response);
// currently only supports CURLOPT_SSL_VERIFYPEER_OFF
// and CURLOPT_SSL_VERIFYHOST_OFF
void SetCurlOptions(std::vector<std::string> const& args);
void SetUseHttp10On() { this->UseHttp10 = true;}
void SetTimeOutSeconds(int s) { this->TimeOutSeconds = s;}
protected:
void SetProxyType();
bool InitCurl();
private:
cmCTest* CTest;
CURL* Curl;
std::string HTTPProxyAuth;
std::string HTTPProxy;
curl_proxytype HTTPProxyType;
bool VerifyHostOff;
bool VerifyPeerOff;
bool UseHttp10;
int TimeOutSeconds;
};
#endif

View File

@ -27,7 +27,8 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
= this->Makefile->GetDefinition("CTEST_TRIGGER_SITE");
bool ctestDropSiteCDash
= this->Makefile->IsOn("CTEST_DROP_SITE_CDASH");
const char* ctestProjectName
= this->Makefile->GetDefinition("CTEST_PROJECT_NAME");
if ( !ctestDropMethod )
{
ctestDropMethod = "http";
@ -43,7 +44,7 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
// error: CDash requires CTEST_DROP_LOCATION definition
// in CTestConfig.cmake
}
this->CTest->SetCTestConfiguration("ProjectName", ctestProjectName);
this->CTest->SetCTestConfiguration("DropMethod", ctestDropMethod);
this->CTest->SetCTestConfiguration("DropSite", ctestDropSite);
this->CTest->SetCTestConfiguration("DropLocation", ctestDropLocation);
@ -144,6 +145,13 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
static_cast<cmCTestSubmitHandler*>(handler)->SetOption("InternalTest",
this->InternalTest ? "ON" : "OFF");
if(this->CDashUploadFile.size())
{
static_cast<cmCTestSubmitHandler*>(handler)->
SetOption("CDashUploadFile", this->CDashUploadFile.c_str());
static_cast<cmCTestSubmitHandler*>(handler)->
SetOption("CDashUploadType", this->CDashUploadType.c_str());
}
return handler;
}
@ -178,6 +186,16 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
return true;
}
if(arg == "CDASH_UPLOAD")
{
this->ArgumentDoing = ArgumentDoingCDashUpload;
return true;
}
if(arg == "CDASH_UPLOAD_TYPE")
{
this->ArgumentDoing = ArgumentDoingCDashUploadType;
return true;
}
if(arg == "INTERNAL_TEST_CHECKSUM")
{
this->InternalTest = true;
@ -240,6 +258,17 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg)
return true;
}
if(this->ArgumentDoing == ArgumentDoingCDashUpload)
{
this->CDashUploadFile = arg;
return true;
}
if(this->ArgumentDoing == ArgumentDoingCDashUploadType)
{
this->CDashUploadType = arg;
return true;
}
// Look for other arguments.
return this->Superclass::CheckArgumentValue(arg);
}

View File

@ -64,6 +64,8 @@ protected:
ArgumentDoingFiles,
ArgumentDoingRetryDelay,
ArgumentDoingRetryCount,
ArgumentDoingCDashUpload,
ArgumentDoingCDashUploadType,
ArgumentDoingLast2
};
@ -74,6 +76,8 @@ protected:
cmCTest::SetOfStrings Files;
std::string RetryCount;
std::string RetryDelay;
std::string CDashUploadFile;
std::string CDashUploadType;
};

View File

@ -10,7 +10,8 @@
See the License for more information.
============================================================================*/
#include "cmCTestSubmitHandler.h"
#include "cmCTestScriptHandler.h"
#include "cmake.h"
#include "cmSystemTools.h"
#include "cmVersion.h"
#include "cmGeneratedFileStream.h"
@ -23,8 +24,10 @@
// For XML-RPC submission
#include "cm_xmlrpc.h"
#include <cm_jsoncpp_reader.h>
// For curl submission
#include "cm_curl.h"
#include "cmCTestCurl.h"
#include <sys/stat.h>
@ -1055,9 +1058,165 @@ bool cmCTestSubmitHandler::SubmitUsingXMLRPC(std::string const&,
}
#endif
void cmCTestSubmitHandler::ConstructCDashURL(std::string& dropMethod,
std::string& url)
{
dropMethod = this->CTest->GetCTestConfiguration("DropMethod");
url = dropMethod;
url += "://";
if ( this->CTest->GetCTestConfiguration("DropSiteUser").size() > 0 )
{
url += this->CTest->GetCTestConfiguration("DropSiteUser");
cmCTestLog(this->CTest, HANDLER_OUTPUT,
this->CTest->GetCTestConfiguration("DropSiteUser").c_str());
if ( this->CTest->GetCTestConfiguration("DropSitePassword").size() > 0 )
{
url += ":" + this->CTest->GetCTestConfiguration("DropSitePassword");
cmCTestLog(this->CTest, HANDLER_OUTPUT, ":******");
}
url += "@";
}
url += this->CTest->GetCTestConfiguration("DropSite") +
this->CTest->GetCTestConfiguration("DropLocation");
}
int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file,
std::string const& typeString)
{
if(!cmSystemTools::FileExists(file))
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Upload file not found: " << file << "\n");
return -1;
}
cmCTestCurl curl(this->CTest);
std::string curlopt(this->CTest->GetCTestConfiguration("CurlOptions"));
std::vector<std::string> args;
cmSystemTools::ExpandListArgument(curlopt, args);
curl.SetCurlOptions(args);
curl.SetTimeOutSeconds(SUBMIT_TIMEOUT_IN_SECONDS_DEFAULT);
std::string dropMethod;
std::string url;
this->ConstructCDashURL(dropMethod, url);
std::string::size_type pos = url.find("submit.php?");
url = url.substr(0, pos+10);
if ( ! (dropMethod == "http" || dropMethod == "https" ) )
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Only http and https are supported for CDASH_UPLOAD\n");
return -1;
}
char md5sum[33];
md5sum[32] = 0;
cmSystemTools::ComputeFileMD5(file, md5sum);
// 1. request the buildid and check to see if the file
// has already been uploaded
// TODO I added support for subproject. You would need to add
// a "&subproject=subprojectname" to the first POST.
cmCTestScriptHandler* ch =
static_cast<cmCTestScriptHandler*>(this->CTest->GetHandler("script"));
cmake* cm = ch->GetCMake();
const char* subproject = cm->GetProperty("SubProject", cmProperty::GLOBAL);
std::ostringstream str;
str << "project="
<< this->CTest->GetCTestConfiguration("ProjectName") << "&";
if(subproject)
{
str << "subproject=" << subproject << "&";
}
str << "stamp=" << this->CTest->GetCurrentTag() << "-"
<< this->CTest->GetTestModelString() << "&"
<< "model=" << this->CTest->GetTestModelString() << "&"
<< "build=" << this->CTest->GetCTestConfiguration("BuildName") << "&"
<< "site=" << this->CTest->GetCTestConfiguration("Site") << "&"
<< "track=" << this->CTest->GetTestModelString() << "&"
<< "starttime=" << (int)cmSystemTools::GetTime() << "&"
<< "endtime=" << (int)cmSystemTools::GetTime() << "&"
<< "datafilesmd5[0]=" << md5sum << "&"
<< "type=" << typeString;
std::string fields = str.str();
cmCTestLog(this->CTest, DEBUG, "fields: " << fields << "\nurl:"
<< url << "\nfile: " << file << "\n");
std::string response;
if(!curl.HttpRequest(url, fields, response))
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Error in HttpRequest\n" << response);
return -1;
}
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"Request upload response: [" << response << "]\n");
Json::Value json;
Json::Reader reader;
if(!reader.parse(response, json))
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"error parsing json string [" << response << "]\n"
<< reader.getFormattedErrorMessages() << "\n");
return -1;
}
if(json["status"].asInt() != 0)
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Bad status returned from CDash: "
<< json["status"].asInt());
return -1;
}
if(json["datafilesmd5"].isArray())
{
int datares = json["datafilesmd5"][0].asInt();
if(datares == 1)
{
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"File already exists on CDash, skip upload "
<< file << "\n");
return 0;
}
}
else
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"bad datafilesmd5 value in response "
<< response << "\n");
return -1;
}
std::string upload_as = cmSystemTools::GetFilenameName(file);
std::ostringstream fstr;
fstr << "type=" << typeString << "&"
<< "md5=" << md5sum << "&"
<< "filename=" << upload_as << "&"
<< "buildid=" << json["buildid"].asString();
if(!curl.UploadFile(file, url, fstr.str(), response))
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"error uploading to CDash. "
<< file << " " << url << " " << fstr.str());
return -1;
}
if(!reader.parse(response, json))
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"error parsing json string [" << response << "]\n"
<< reader.getFormattedErrorMessages() << "\n");
return -1;
}
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"Upload file response: [" << response << "]\n");
return 0;
}
//----------------------------------------------------------------------------
int cmCTestSubmitHandler::ProcessHandler()
{
const char* cdashUploadFile = this->GetOption("CDashUploadFile");
const char* cdashUploadType = this->GetOption("CDashUploadType");
if(cdashUploadFile && cdashUploadType)
{
return this->HandleCDashUploadFile(std::string(cdashUploadFile),
std::string(cdashUploadType));
}
std::string iscdash = this->CTest->GetCTestConfiguration("IsCDash");
// cdash does not need to trigger so just return true
if(!iscdash.empty())

View File

@ -41,6 +41,11 @@ public:
/** Specify a set of files to submit. */
void SelectFiles(cmCTest::SetOfStrings const& files);
// handle the cdash file upload protocol
int HandleCDashUploadFile(std::string const& file, std::string const& type);
void ConstructCDashURL(std::string& dropMethod, std::string& url);
private:
void SetLogFile(std::ostream* ost) { this->LogFile = ost; }