kolan
/
CMake
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge topic 'ExternalProject-improve-retry' 

89113e12 ExternalProject: Re-implement download logic as a dedicated script
e7d5e4b4 ExternalProject: Re-implement download verification as a dedicated script
ebcc7027 ExternalProject: Avoid repeating download verification
33218f6a ExternalProject: Remove unused verify script logic
e5409d1e ExternalProject: Remove unused 'retries' argument from verify script
d610407c ExternalProject: Use uppercase placeholders for script generation
This commit is contained in:
Brad King 2016-05-20 09:15:55 -04:00 committed by CMake Topic Stage
parent acd039ae0e 89113e125d
commit e07f7e6cd5
3 changed files with 262 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
Modules/ExternalProject-download.cmake.in Normal file
View File

@ -0,0 +1,161 @@
#=============================================================================
# Copyright 2008-2013 Kitware, Inc.
# Copyright 2016 Ruslan Baratov
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
cmake_minimum_required(VERSION 3.5)
function(check_file_hash has_hash hash_is_good)
if("${has_hash}" STREQUAL "")
message(FATAL_ERROR "has_hash Can't be empty")
endif()
if("${hash_is_good}" STREQUAL "")
message(FATAL_ERROR "hash_is_good Can't be empty")
endif()
if("@ALGO@" STREQUAL "")
# No check
set("${has_hash}" FALSE PARENT_SCOPE)
set("${hash_is_good}" FALSE PARENT_SCOPE)
return()
endif()
set("${has_hash}" TRUE PARENT_SCOPE)
message(STATUS "verifying file...
file='@LOCAL@'")
file("@ALGO@" "@LOCAL@" actual_value)
if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
set("${hash_is_good}" FALSE PARENT_SCOPE)
message(STATUS "@ALGO@ hash of
@LOCAL@
does not match expected value
expected: '@EXPECT_VALUE@'
actual: '${actual_value}'")
else()
set("${hash_is_good}" TRUE PARENT_SCOPE)
endif()
endfunction()
function(sleep_before_download attempt)
if(attempt EQUAL 0)
return()
endif()
if(attempt EQUAL 1)
message(STATUS "Retrying...")
return()
endif()
set(sleep_seconds 0)
if(attempt EQUAL 2)
set(sleep_seconds 5)
elseif(attempt EQUAL 3)
set(sleep_seconds 5)
elseif(attempt EQUAL 4)
set(sleep_seconds 15)
elseif(attempt EQUAL 5)
set(sleep_seconds 60)
elseif(attempt EQUAL 6)
set(sleep_seconds 90)
elseif(attempt EQUAL 7)
set(sleep_seconds 300)
else()
set(sleep_seconds 1200)
endif()
message(STATUS "Retry after ${sleep_seconds} seconds (attempt #${attempt}) ...")
execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${sleep_seconds}")
endfunction()
if("@LOCAL@" STREQUAL "")
message(FATAL_ERROR "LOCAL can't be empty")
endif()
if("@REMOTE@" STREQUAL "")
message(FATAL_ERROR "REMOTE can't be empty")
endif()
if(EXISTS "@LOCAL@")
check_file_hash(has_hash hash_is_good)
if(has_hash)
if(hash_is_good)
message(STATUS "File already exists and hash match (skip download):
file='@LOCAL@'
@ALGO@='@EXPECT_VALUE@'"
)
return()
else()
message(STATUS "File already exists but hash mismatch. Removing...")
file(REMOVE "@LOCAL@")
endif()
else()
message(STATUS "File already exists but no hash specified (use URL_HASH):
file='@LOCAL@'
Old file will be removed and new file downloaded from URL."
)
file(REMOVE "@LOCAL@")
endif()
endif()
set(retry_number 5)
foreach(i RANGE ${retry_number})
sleep_before_download(${i})
message(STATUS "downloading...
src='@REMOTE@'
dst='@LOCAL@'
timeout='@TIMEOUT_MSG@'")
@TLS_VERIFY_CODE@
@TLS_CAINFO_CODE@
file(
DOWNLOAD
"@REMOTE@" "@LOCAL@"
@SHOW_PROGRESS@
@TIMEOUT_ARGS@
STATUS status
LOG log
)
list(GET status 0 status_code)
list(GET status 1 status_string)
if(status_code EQUAL 0)
check_file_hash(has_hash hash_is_good)
if(has_hash AND NOT hash_is_good)
message(STATUS "Hash mismatch, removing...")
file(REMOVE "@LOCAL@")
else()
message(STATUS "Downloading... done")
return()
endif()
else()
message("error: downloading '@REMOTE@' failed
status_code: ${status_code}
status_string: ${status_string}
log:
--- LOG BEGIN ---
${log}
--- LOG END ---"
)
endif()
endforeach()
message(FATAL_ERROR "Downloading failed")

48
Modules/ExternalProject-verify.cmake.in Normal file
View File

@ -0,0 +1,48 @@
#=============================================================================
# Copyright 2008-2013 Kitware, Inc.
# Copyright 2016 Ruslan Baratov
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
cmake_minimum_required(VERSION 3.5)
if("@LOCAL@" STREQUAL "")
message(FATAL_ERROR "LOCAL can't be empty")
endif()
if(NOT EXISTS "@LOCAL@")
message(FATAL_ERROR "File not found: @LOCAL@")
endif()
if("@ALGO@" STREQUAL "")
message(WARNING "File will not be verified since no URL_HASH specified")
return()
endif()
if("@EXPECT_VALUE@" STREQUAL "")
message(FATAL_ERROR "EXPECT_VALUE can't be empty")
endif()
message(STATUS "verifying file...
file='@LOCAL@'")
file("@ALGO@" "@LOCAL@" actual_value)
if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
message(FATAL_ERROR "error: @ALGO@ hash of
@LOCAL@
does not match expected value
expected: '@EXPECT_VALUE@'
actual: '${actual_value}'
")
endif()
message(STATUS "verifying file... done")

151
Modules/ExternalProject.cmake
View File

@ -377,6 +377,7 @@ file::
#============================================================================= #=============================================================================
# Copyright 2008-2013 Kitware, Inc. # Copyright 2008-2013 Kitware, Inc.
# Copyright 2016 Ruslan Baratov
# #
# Distributed under the OSI-approved BSD License (the "License"); # Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details. # see accompanying file Copyright.txt for details.
@ -418,6 +419,9 @@ endif()
set(_ep_hash_algos "MD5|SHA1|SHA224|SHA256|SHA384|SHA512") set(_ep_hash_algos "MD5|SHA1|SHA224|SHA256|SHA384|SHA512")
set(_ep_hash_regex "^(${_ep_hash_algos})=([0-9A-Fa-f]+)$") set(_ep_hash_regex "^(${_ep_hash_algos})=([0-9A-Fa-f]+)$")
set(_ExternalProject_SELF "${CMAKE_CURRENT_LIST_FILE}")
get_filename_component(_ExternalProject_SELF_DIR "${_ExternalProject_SELF}" PATH)
function(_ep_parse_arguments f name ns args) function(_ep_parse_arguments f name ns args)
# Transfer the arguments to this function into target properties for the # Transfer the arguments to this function into target properties for the
# new custom target we just added so that we can set up all the build steps # new custom target we just added so that we can set up all the build steps
@ -847,43 +851,38 @@ endif()
endfunction(_ep_write_gitupdate_script) endfunction(_ep_write_gitupdate_script)
function(_ep_write_downloadfile_script script_filename remote local timeout no_progress hash tls_verify tls_cainfo) function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_progress hash tls_verify tls_cainfo)
if(timeout) if(timeout)
set(timeout_args TIMEOUT ${timeout}) set(TIMEOUT_ARGS TIMEOUT ${timeout})
set(timeout_msg "${timeout} seconds") set(TIMEOUT_MSG "${timeout} seconds")
else() else()
set(timeout_args "# no TIMEOUT") set(TIMEOUT_ARGS "# no TIMEOUT")
set(timeout_msg "none") set(TIMEOUT_MSG "none")
endif() endif()
if(no_progress) if(no_progress)
set(show_progress "") set(SHOW_PROGRESS "")
else() else()
set(show_progress "SHOW_PROGRESS") set(SHOW_PROGRESS "SHOW_PROGRESS")
endif() endif()
if("${hash}" MATCHES "${_ep_hash_regex}") if("${hash}" MATCHES "${_ep_hash_regex}")
string(CONCAT hash_check set(ALGO "${CMAKE_MATCH_1}")
"if(EXISTS \"${local}\")\n" set(EXPECT_VALUE "${CMAKE_MATCH_2}")
" file(\"${CMAKE_MATCH_1}\" \"${local}\" hash_value)\n"
" if(\"x\${hash_value}\" STREQUAL \"x${CMAKE_MATCH_2}\")\n"
" return()\n"
" endif()\n"
"endif()\n"
)
else() else()
set(hash_check "") set(ALGO "")
set(EXPECT_VALUE "")
endif() endif()
set(tls_verify_code "") set(TLS_VERIFY_CODE "")
set(tls_cainfo_code "") set(TLS_CAINFO_CODE "")
# check for curl globals in the project # check for curl globals in the project
if(DEFINED CMAKE_TLS_VERIFY) if(DEFINED CMAKE_TLS_VERIFY)
set(tls_verify_code "set(CMAKE_TLS_VERIFY ${CMAKE_TLS_VERIFY})") set(TLS_VERIFY_CODE "set(CMAKE_TLS_VERIFY ${CMAKE_TLS_VERIFY})")
endif() endif()
if(DEFINED CMAKE_TLS_CAINFO) if(DEFINED CMAKE_TLS_CAINFO)
set(tls_cainfo_code "set(CMAKE_TLS_CAINFO \"${CMAKE_TLS_CAINFO}\")") set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${CMAKE_TLS_CAINFO}\")")
endif() endif()
# now check for curl locals so that the local values # now check for curl locals so that the local values
@ -892,91 +891,49 @@ function(_ep_write_downloadfile_script script_filename remote local timeout no_p
# check for tls_verify argument # check for tls_verify argument
string(LENGTH "${tls_verify}" tls_verify_len) string(LENGTH "${tls_verify}" tls_verify_len)
if(tls_verify_len GREATER 0) if(tls_verify_len GREATER 0)
set(tls_verify_code "set(CMAKE_TLS_VERIFY ${tls_verify})") set(TLS_VERIFY_CODE "set(CMAKE_TLS_VERIFY ${tls_verify})")
endif() endif()
# check for tls_cainfo argument # check for tls_cainfo argument
string(LENGTH "${tls_cainfo}" tls_cainfo_len) string(LENGTH "${tls_cainfo}" tls_cainfo_len)
if(tls_cainfo_len GREATER 0) if(tls_cainfo_len GREATER 0)
set(tls_cainfo_code "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")") set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")")
endif() endif()
file(WRITE ${script_filename} # Used variables:
"${hash_check}message(STATUS \"downloading... # * TLS_VERIFY_CODE
src='${remote}' # * TLS_CAINFO_CODE
dst='${local}' # * ALGO
timeout='${timeout_msg}'\") # * EXPECT_VALUE
# * REMOTE
${tls_verify_code} # * LOCAL
${tls_cainfo_code} # * SHOW_PROGRESS
# * TIMEOUT_ARGS
file(DOWNLOAD # * TIMEOUT_MSG
\"${remote}\" configure_file(
\"${local}\" "${_ExternalProject_SELF_DIR}/ExternalProject-download.cmake.in"
${show_progress} "${script_filename}"
${timeout_args} @ONLY
STATUS status )
LOG log)
list(GET status 0 status_code)
list(GET status 1 status_string)
if(NOT status_code EQUAL 0)
message(FATAL_ERROR \"error: downloading '${remote}' failed
status_code: \${status_code}
status_string: \${status_string}
log: \${log}
\")
endif()
message(STATUS \"downloading... done\")
"
)
endfunction() endfunction()
function(_ep_write_verifyfile_script script_filename LOCAL hash)
function(_ep_write_verifyfile_script script_filename local hash retries download_script)
if("${hash}" MATCHES "${_ep_hash_regex}") if("${hash}" MATCHES "${_ep_hash_regex}")
set(algo "${CMAKE_MATCH_1}") set(ALGO "${CMAKE_MATCH_1}")
string(TOLOWER "${CMAKE_MATCH_2}" expect_value) string(TOLOWER "${CMAKE_MATCH_2}" EXPECT_VALUE)
set(script_content "set(expect_value \"${expect_value}\")
set(attempt 0)
set(succeeded 0)
while(\${attempt} LESS ${retries} OR \${attempt} EQUAL ${retries} AND NOT \${succeeded})
file(${algo} \"\${file}\" actual_value)
if(\"\${actual_value}\" STREQUAL \"\${expect_value}\")
set(succeeded 1)
elseif(\${attempt} LESS ${retries})
message(STATUS \"${algo} hash of \${file}
does not match expected value
expected: \${expect_value}
actual: \${actual_value}
Retrying download.
\")
file(REMOVE \"\${file}\")
execute_process(COMMAND \${CMAKE_COMMAND} -P \"${download_script}\")
endif()
math(EXPR attempt \"\${attempt} + 1\")
endwhile()
if(\${succeeded})
message(STATUS \"verifying file... done\")
else()
message(FATAL_ERROR \"error: ${algo} hash of
\${file}
does not match expected value
expected: \${expect_value}
actual: \${actual_value}
\")
endif()")
else() else()
set(script_content "message(STATUS \"verifying file... warning: did not verify file - no URL_HASH specified?\")") set(ALGO "")
set(EXPECT_VALUE "")
endif() endif()
file(WRITE ${script_filename} "set(file \"${local}\")
message(STATUS \"verifying file... # Used variables:
file='\${file}'\") # * ALGO
${script_content} # * EXPECT_VALUE
") # * LOCAL
configure_file(
"${_ExternalProject_SELF_DIR}/ExternalProject-verify.cmake.in"
"${script_filename}"
@ONLY
)
endfunction() endfunction()
@ -1898,8 +1855,6 @@ function(_ep_add_download_command name)
set(repository "external project URL") set(repository "external project URL")
set(module "${url}") set(module "${url}")
set(tag "${hash}") set(tag "${hash}")
set(retries 0)
set(download_script "")
configure_file( configure_file(
"${CMAKE_ROOT}/Modules/RepositoryInfo.txt.in" "${CMAKE_ROOT}/Modules/RepositoryInfo.txt.in"
"${stamp_dir}/${name}-urlinfo.txt" "${stamp_dir}/${name}-urlinfo.txt"
@ -1937,13 +1892,13 @@ function(_ep_add_download_command name)
_ep_write_downloadfile_script("${download_script}" "${url}" "${file}" "${timeout}" "${no_progress}" "${hash}" "${tls_verify}" "${tls_cainfo}") _ep_write_downloadfile_script("${download_script}" "${url}" "${file}" "${timeout}" "${no_progress}" "${hash}" "${tls_verify}" "${tls_cainfo}")
set(cmd ${CMAKE_COMMAND} -P "${download_script}" set(cmd ${CMAKE_COMMAND} -P "${download_script}"
COMMAND) COMMAND)
set(retries 3)
if (no_extract) if (no_extract)
set(steps "download and verify") set(steps "download and verify")
else () else ()
set(steps "download, verify and extract") set(steps "download, verify and extract")
endif () endif ()
set(comment "Performing download step (${steps}) for '${name}'") set(comment "Performing download step (${steps}) for '${name}'")
file(WRITE "${stamp_dir}/verify-${name}.cmake" "") # already verified by 'download_script'
else() else()
set(file "${url}") set(file "${url}")
if (no_extract) if (no_extract)
@ -1952,8 +1907,8 @@ function(_ep_add_download_command name)
set(steps "verify and extract") set(steps "verify and extract")
endif () endif ()
set(comment "Performing download step (${steps}) for '${name}'") set(comment "Performing download step (${steps}) for '${name}'")
_ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${hash}")
endif() endif()
_ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${hash}" "${retries}" "${download_script}")
list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/verify-${name}.cmake list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/verify-${name}.cmake
COMMAND) COMMAND)
if (NOT no_extract) if (NOT no_extract)