Merge branch 'improve-file-download'

Conflicts:
	Modules/ExternalProject.cmake
This commit is contained in:
Brad King 2010-06-07 14:26:24 -04:00
commit 0d07e4379e
7 changed files with 373 additions and 29 deletions

View File

@ -21,6 +21,7 @@
# [GIT_REPOSITORY url] # URL of git repo
# [GIT_TAG tag] # Git branch name, commit id or tag
# [URL /.../src.tgz] # Full path or URL of source
# [URL_MD5 md5] # MD5 checksum of file at URL
# [TIMEOUT seconds] # Time allowed for file download operations
# #--Update/Patch step----------
# [UPDATE_COMMAND cmd...] # Source work-tree update command
@ -115,19 +116,19 @@
# License text for the above reference.)
# Pre-compute a regex to match documented keywords for each command.
file(STRINGS "${CMAKE_CURRENT_LIST_FILE}" lines LIMIT_COUNT 100
REGEX "^# ( \\[[A-Z_]+ [^]]*\\] +#.*$|[A-Za-z_]+\\()")
file(STRINGS "${CMAKE_CURRENT_LIST_FILE}" lines LIMIT_COUNT 103
REGEX "^# ( \\[[A-Z0-9_]+ [^]]*\\] +#.*$|[A-Za-z0-9_]+\\()")
foreach(line IN LISTS lines)
if("${line}" MATCHES "^# [A-Za-z_]+\\(")
if("${line}" MATCHES "^# [A-Za-z0-9_]+\\(")
if(_ep_func)
set(_ep_keywords_${_ep_func} "${_ep_keywords_${_ep_func}})$")
endif()
string(REGEX REPLACE "^# ([A-Za-z_]+)\\(.*" "\\1" _ep_func "${line}")
string(REGEX REPLACE "^# ([A-Za-z0-9_]+)\\(.*" "\\1" _ep_func "${line}")
#message("function [${_ep_func}]")
set(_ep_keywords_${_ep_func} "^(")
set(_ep_keyword_sep)
else()
string(REGEX REPLACE "^# \\[([A-Z_]+) .*" "\\1" _ep_key "${line}")
string(REGEX REPLACE "^# \\[([A-Z0-9_]+) .*" "\\1" _ep_key "${line}")
#message(" keyword [${_ep_key}]")
set(_ep_keywords_${_ep_func}
"${_ep_keywords_${_ep_func}}${_ep_keyword_sep}${_ep_key}")
@ -152,7 +153,7 @@ function(_ep_parse_arguments f name ns args)
foreach(arg IN LISTS args)
set(is_value 1)
if(arg MATCHES "^[A-Z][A-Z_][A-Z_]+$" AND
if(arg MATCHES "^[A-Z][A-Z0-9_][A-Z0-9_]+$" AND
NOT ((arg STREQUAL "${key}") AND (key STREQUAL "COMMAND")) AND
NOT arg MATCHES "^(TRUE|FALSE)$")
if(_ep_keywords_${f} AND arg MATCHES "${_ep_keywords_${f}}")
@ -264,7 +265,7 @@ endif()
endfunction(_ep_write_gitclone_script)
function(_ep_write_downloadfile_script script_filename remote local timeout)
function(_ep_write_downloadfile_script script_filename remote local timeout md5)
if(timeout)
set(timeout_args TIMEOUT ${timeout})
set(timeout_msg "${timeout} seconds")
@ -273,6 +274,12 @@ function(_ep_write_downloadfile_script script_filename remote local timeout)
set(timeout_msg "none")
endif()
if(md5)
set(md5_args EXPECTED_MD5 ${md5})
else()
set(md5_args "# no EXPECTED_MD5")
endif()
file(WRITE ${script_filename}
"message(STATUS \"downloading...
src='${remote}'
@ -282,6 +289,8 @@ function(_ep_write_downloadfile_script script_filename remote local timeout)
file(DOWNLOAD
\"${remote}\"
\"${local}\"
SHOW_PROGRESS
${md5_args}
${timeout_args}
STATUS status
LOG log)
@ -304,6 +313,51 @@ message(STATUS \"downloading... done\")
endfunction(_ep_write_downloadfile_script)
function(_ep_write_verifyfile_script script_filename local md5)
file(WRITE ${script_filename}
"message(STATUS \"verifying file...
file='${local}'\")
set(verified 0)
# If an expected md5 checksum exists, compare against it:
#
if(NOT \"${md5}\" STREQUAL \"\")
execute_process(COMMAND \${CMAKE_COMMAND} -E md5sum \"${local}\"
OUTPUT_VARIABLE ov
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE rv)
if(NOT rv EQUAL 0)
message(FATAL_ERROR \"error: computing md5sum of '${local}' failed\")
endif()
string(REGEX MATCH \"^([0-9A-Fa-f]+)\" md5_actual \"\${ov}\")
string(TOLOWER \"\${md5_actual}\" md5_actual)
string(TOLOWER \"${md5}\" md5)
if(NOT \"\${md5}\" STREQUAL \"\${md5_actual}\")
message(FATAL_ERROR \"error: md5sum of '${local}' does not match expected value
md5_expected: \${md5}
md5_actual: \${md5_actual}
\")
endif()
set(verified 1)
endif()
if(verified)
message(STATUS \"verifying file... done\")
else()
message(STATUS \"verifying file... warning: did not verify file - no URL_MD5 checksum argument? corrupt file?\")
endif()
"
)
endfunction(_ep_write_verifyfile_script)
function(_ep_write_extractfile_script script_filename filename directory)
set(args "")
@ -797,9 +851,10 @@ function(_ep_add_download_command name)
list(APPEND depends ${stamp_dir}/${name}-gitinfo.txt)
elseif(url)
get_filename_component(work_dir "${source_dir}" PATH)
get_property(md5 TARGET ${name} PROPERTY _EP_URL_MD5)
set(repository "external project URL")
set(module "${url}")
set(tag "")
set(tag "${md5}")
configure_file(
"${CMAKE_ROOT}/Modules/RepositoryInfo.txt.in"
"${stamp_dir}/${name}-urlinfo.txt"
@ -820,14 +875,16 @@ function(_ep_add_download_command name)
endif()
set(file ${download_dir}/${fname})
get_property(timeout TARGET ${name} PROPERTY _EP_TIMEOUT)
_ep_write_downloadfile_script("${stamp_dir}/download-${name}.cmake" "${url}" "${file}" "${timeout}")
_ep_write_downloadfile_script("${stamp_dir}/download-${name}.cmake" "${url}" "${file}" "${timeout}" "${md5}")
set(cmd ${CMAKE_COMMAND} -P ${stamp_dir}/download-${name}.cmake
COMMAND)
set(comment "Performing download step (download and extract) for '${name}'")
set(comment "Performing download step (download, verify and extract) for '${name}'")
else()
set(file "${url}")
set(comment "Performing download step (extract) for '${name}'")
set(comment "Performing download step (verify and extract) for '${name}'")
endif()
_ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${md5}")
list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/verify-${name}.cmake)
# TODO: Support other archive formats.
_ep_write_extractfile_script("${stamp_dir}/extract-${name}.cmake" "${file}" "${source_dir}")
list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/extract-${name}.cmake)

View File

@ -2450,7 +2450,8 @@ namespace{
fout->write(chPtr, realsize);
return realsize;
}
static size_t
cmFileCommandCurlDebugCallback(CURL *, curl_infotype, char *chPtr,
size_t size, void *data)
@ -2463,6 +2464,72 @@ namespace{
}
class cURLProgressHelper
{
public:
cURLProgressHelper(cmFileCommand *fc)
{
this->CurrentPercentage = -1;
this->FileCommand = fc;
}
bool UpdatePercentage(double value, double total, std::string &status)
{
int OldPercentage = this->CurrentPercentage;
if (0.0 == total)
{
this->CurrentPercentage = 100;
}
else
{
this->CurrentPercentage = static_cast<int>(value/total*100.0 + 0.5);
}
bool updated = (OldPercentage != this->CurrentPercentage);
if (updated)
{
cmOStringStream oss;
oss << "[download " << this->CurrentPercentage << "% complete]";
status = oss.str();
}
return updated;
}
cmFileCommand *GetFileCommand()
{
return this->FileCommand;
}
private:
int CurrentPercentage;
cmFileCommand *FileCommand;
};
static int
cmFileCommandCurlProgressCallback(void *clientp,
double dltotal, double dlnow,
double ultotal, double ulnow)
{
cURLProgressHelper *helper =
reinterpret_cast<cURLProgressHelper *>(clientp);
static_cast<void>(ultotal);
static_cast<void>(ulnow);
std::string status;
if (helper->UpdatePercentage(dlnow, dltotal, status))
{
cmFileCommand *fc = helper->GetFileCommand();
cmMakefile *mf = fc->GetMakefile();
mf->DisplayStatus(status.c_str(), -1);
}
return 0;
}
}
#endif
@ -2476,8 +2543,8 @@ namespace {
cURLEasyGuard(CURL * easy)
: Easy(easy)
{}
~cURLEasyGuard(void)
~cURLEasyGuard(void)
{
if (this->Easy)
{
@ -2498,6 +2565,7 @@ namespace {
}
#endif
bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string>
const& args)
@ -2515,9 +2583,13 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
++i;
std::string file = *i;
++i;
long timeout = 0;
std::string verboseLog;
std::string statusVar;
std::string expectedMD5sum;
bool showProgress = false;
while(i != args.end())
{
if(*i == "TIMEOUT")
@ -2556,9 +2628,65 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
}
statusVar = *i;
}
else if(*i == "EXPECTED_MD5")
{
++i;
if( i == args.end())
{
this->SetError("FILE(DOWNLOAD url file EXPECTED_MD5 sum) missing "
"sum value for EXPECTED_MD5.");
return false;
}
expectedMD5sum = cmSystemTools::LowerCase(*i);
}
else if(*i == "SHOW_PROGRESS")
{
showProgress = true;
}
++i;
}
// If file exists already, and caller specified an expected md5 sum,
// and the existing file already has the expected md5 sum, then simply
// return.
//
if(cmSystemTools::FileExists(file.c_str()) &&
!expectedMD5sum.empty())
{
char computedMD5[32];
if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
{
this->SetError("FILE(DOWNLOAD ) error; cannot compute MD5 sum on "
"pre-existing file");
return false;
}
std::string actualMD5sum = cmSystemTools::LowerCase(
std::string(computedMD5, 32));
if (expectedMD5sum == actualMD5sum)
{
this->Makefile->DisplayStatus(
"FILE(DOWNLOAD ) returning early: file already exists with "
"expected MD5 sum", -1);
if(statusVar.size())
{
cmOStringStream result;
result << (int)0 << ";\""
"returning early: file already exists with expected MD5 sum\"";
this->Makefile->AddDefinition(statusVar.c_str(),
result.str().c_str());
}
return true;
}
}
// Make sure parent directory exists so we can write to the file
// as we receive downloaded bits from curl...
//
std::string dir = cmSystemTools::GetFilenamePath(file.c_str());
if(!cmSystemTools::FileExists(dir.c_str()) &&
!cmSystemTools::MakeDirectory(dir.c_str()))
@ -2577,6 +2705,7 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
"file for write.");
return false;
}
::CURL *curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
curl = ::curl_easy_init();
@ -2592,28 +2721,31 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set url: ";
errstring += ::curl_easy_strerror(res);
std::string errstring = "FILE(DOWNLOAD ) error; cannot set url: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
cmFileCommandWriteMemoryCallback);
cmFileCommandWriteMemoryCallback);
if (res != CURLE_OK)
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set write function: ";
errstring += ::curl_easy_strerror(res);
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set write function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
cmFileCommandCurlDebugCallback);
cmFileCommandCurlDebugCallback);
if (res != CURLE_OK)
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set debug function: ";
errstring += ::curl_easy_strerror(res);
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set debug function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
@ -2625,14 +2757,25 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set write data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set write data: ";
std::string errstring = "FILE(DOWNLOAD ) error; cannot set debug data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set follow-redirect option: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
@ -2644,24 +2787,70 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set verbose: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
if(timeout > 0)
{
res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout );
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set verbose: ";
std::string errstring = "FILE(DOWNLOAD ) error; cannot set timeout: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
// Need the progress helper's scope to last through the duration of
// the curl_easy_perform call... so this object is declared at function
// scope intentionally, rather than inside the "if(showProgress)"
// block...
//
cURLProgressHelper helper(this);
if(showProgress)
{
res = ::curl_easy_setopt(curl,
CURLOPT_NOPROGRESS, 0);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set noprogress value: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl,
CURLOPT_PROGRESSFUNCTION, cmFileCommandCurlProgressCallback);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set progress function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl,
CURLOPT_PROGRESSDATA, reinterpret_cast<void*>(&helper));
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set progress data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
res = ::curl_easy_perform(curl);
/* always cleanup */
g_curl.release();
::curl_easy_cleanup(curl);
if(statusVar.size())
{
cmOStringStream result;
@ -2669,7 +2858,44 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
this->Makefile->AddDefinition(statusVar.c_str(),
result.str().c_str());
}
::curl_global_cleanup();
// Explicitly flush/close so we can measure the md5 accurately.
//
fout.flush();
fout.close();
// Verify MD5 sum if requested:
//
if (!expectedMD5sum.empty())
{
char computedMD5[32];
if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
{
this->SetError("FILE(DOWNLOAD ) error; cannot compute MD5 sum on "
"downloaded file");
return false;
}
std::string actualMD5sum = cmSystemTools::LowerCase(
std::string(computedMD5, 32));
if (expectedMD5sum != actualMD5sum)
{
cmOStringStream oss;
oss << "FILE(DOWNLOAD ) error; expected and actual MD5 sums differ"
<< std::endl
<< " for file: [" << file << "]" << std::endl
<< " expected MD5 sum: [" << expectedMD5sum << "]" << std::endl
<< " actual MD5 sum: [" << actualMD5sum << "]" << std::endl
;
this->SetError(oss.str().c_str());
return false;
}
}
if(chunkDebug.size())
{
chunkDebug.push_back(0);
@ -2687,6 +2913,7 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
this->Makefile->AddDefinition(verboseLog.c_str(),
&*chunkDebug.begin());
}
return true;
#else
this->SetError("FILE(DOWNLOAD ) "

View File

@ -80,7 +80,8 @@ public:
" file(RELATIVE_PATH variable directory file)\n"
" file(TO_CMAKE_PATH path result)\n"
" file(TO_NATIVE_PATH path result)\n"
" file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log])\n"
" file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log]\n"
" [EXPECTED_MD5 sum] [SHOW_PROGRESS])\n"
"WRITE will write a message into a file called 'filename'. It "
"overwrites the file if it already exists, and creates the file "
"if it does not exist.\n"
@ -152,7 +153,12 @@ public:
"and the second element is a string value for the error. A 0 "
"numeric error means no error in the operation. "
"If TIMEOUT time is specified, the operation will "
"timeout after time seconds, time should be specified as an integer."
"timeout after time seconds, time should be specified as an integer. "
"If EXPECTED_MD5 sum is specified, the operation will verify that the "
"downloaded file's actual md5 sum matches the expected value. If it "
"does not match, the operation fails with an error. "
"If SHOW_PROGRESS is specified, progress information will be printed "
"as status messages until the operation is complete."
"\n"
"The file() command also provides COPY and INSTALL signatures:\n"
" file(<COPY|INSTALL> files... DESTINATION <dir>\n"

View File

@ -28,6 +28,11 @@ AddCMakeTest(Math "")
AddCMakeTest(CMakeMinimumRequired "")
AddCMakeTest(CompilerIdVendor "")
AddCMakeTest(FileDownload "")
set_property(TEST CMake.FileDownload PROPERTY
PASS_REGULAR_EXPRESSION "file already exists with expected MD5 sum"
)
if(HAVE_ELF_H)
AddCMakeTest(ELF "")
endif()

Binary file not shown.

After

Width:  |  Height:  |  Size: 358 B

View File

@ -0,0 +1,41 @@
set(url "file://@CMAKE_CURRENT_SOURCE_DIR@/FileDownloadInput.png")
set(dir "@CMAKE_CURRENT_BINARY_DIR@/downloads")
message(STATUS "FileDownload:1")
file(DOWNLOAD
${url}
${dir}/file1.png
TIMEOUT 2
)
message(STATUS "FileDownload:2")
file(DOWNLOAD
${url}
${dir}/file2.png
TIMEOUT 2
SHOW_PROGRESS
)
# Two calls in a row, exactly the same arguments.
# Since downloaded file should exist already for 2nd call,
# the 2nd call will short-circuit and return early...
#
if(EXISTS ${dir}/file3.png)
file(REMOVE ${dir}/file3.png)
endif()
message(STATUS "FileDownload:3")
file(DOWNLOAD
${url}
${dir}/file3.png
TIMEOUT 2
EXPECTED_MD5 d16778650db435bda3a8c3435c3ff5d1
)
message(STATUS "FileDownload:4")
file(DOWNLOAD
${url}
${dir}/file3.png
TIMEOUT 2
EXPECTED_MD5 d16778650db435bda3a8c3435c3ff5d1
)

View File

@ -64,7 +64,9 @@ ExternalProject_Add(${proj}
SVN_REPOSITORY ""
SVN_REVISION ""
TEST_COMMAND ""
TIMEOUT ""
URL ""
URL_MD5 ""
UPDATE_COMMAND ""
)
@ -95,6 +97,7 @@ endif()
set(proj TutorialStep1-LocalTAR)
ExternalProject_Add(${proj}
URL "${CMAKE_CURRENT_SOURCE_DIR}/Step1.tar"
URL_MD5 a87c5b47c0201c09ddfe1d5738fdb1e3
LIST_SEPARATOR ::
PATCH_COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Step1Patch.cmake
CMAKE_GENERATOR "${CMAKE_GENERATOR}"
@ -106,6 +109,7 @@ ExternalProject_Add(${proj}
set(proj TutorialStep1-LocalNoDirTAR)
ExternalProject_Add(${proj}
URL "${CMAKE_CURRENT_SOURCE_DIR}/Step1NoDir.tar"
URL_MD5 d09e3d370c5c908fa035c30939ee438e
LIST_SEPARATOR @@
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> -G ${CMAKE_GENERATOR} <SOURCE_DIR>
-DTEST_LIST:STRING=1@@2@@3
@ -125,6 +129,7 @@ ExternalProject_Add_Step(${proj} mypatch
set(proj TutorialStep1-LocalTGZ)
ExternalProject_Add(${proj}
URL "${CMAKE_CURRENT_SOURCE_DIR}/Step1.tgz"
URL_MD5 38c648e817339c356f6be00eeed79bd0
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> -G ${CMAKE_GENERATOR} <SOURCE_DIR>
INSTALL_COMMAND ""
)
@ -132,6 +137,7 @@ ExternalProject_Add(${proj}
set(proj TutorialStep1-LocalNoDirTGZ)
ExternalProject_Add(${proj}
URL "${CMAKE_CURRENT_SOURCE_DIR}/Step1NoDir.tgz"
URL_MD5 0b8182edcecdf40bf1c9d71d7d259f78
CMAKE_GENERATOR "${CMAKE_GENERATOR}"
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
INSTALL_COMMAND ""
@ -210,6 +216,7 @@ if(do_cvs_tests)
ExternalProject_Add(${proj}
SOURCE_DIR ${local_cvs_repo}
URL ${CMAKE_CURRENT_SOURCE_DIR}/cvsrepo.tgz
URL_MD5 55fc85825ffdd9ed2597123c68b79f7e
BUILD_COMMAND ""
CONFIGURE_COMMAND "${CVS_EXECUTABLE}" --version
INSTALL_COMMAND ""
@ -308,6 +315,7 @@ if(do_svn_tests)
ExternalProject_Add(${proj}
SOURCE_DIR ${local_svn_repo}
URL ${CMAKE_CURRENT_SOURCE_DIR}/svnrepo.tgz
URL_MD5 2f468be4ed1fa96377fca0cc830819c4
BUILD_COMMAND ""
CONFIGURE_COMMAND "${Subversion_SVN_EXECUTABLE}" --version
INSTALL_COMMAND ""