ExternalProject: Re-implement download logic as a dedicated script
Move the content to a `ExternalProject-download.cmake.in` file and use `configure_file` to generate the final script. Retry logic was not working before because similar script trigger FATAL_ERROR if 'file(DOWNLOAD ...)' exits with nonzero 'status_code'. FATAL_ERROR makes the whole chain of commands stop and '_ep_write_verifyfile_script' retry logic was not used in fact. Default retry number set to 5 with pauses 0, 5, 5, 15, 60 seconds. Some space left for future improvements if needed (90, 300, 1200=20min). Can be controlled by user.
This commit is contained in:
Ruslan Baratov
Brad King
parent
e7d5e4b4bf
commit
89113e125d
|
|
@ -0,0 +1,161 @@
|
|||
#=============================================================================
|
||||
# Copyright 2008-2013 Kitware, Inc.
|
||||
# Copyright 2016 Ruslan Baratov
|
||||
#
|
||||
# Distributed under the OSI-approved BSD License (the "License");
|
||||
# see accompanying file Copyright.txt for details.
|
||||
#
|
||||
# This software is distributed WITHOUT ANY WARRANTY; without even the
|
||||
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See the License for more information.
|
||||
#=============================================================================
|
||||
# (To distribute this file outside of CMake, substitute the full
|
||||
# License text for the above reference.)
|
||||
|
||||
cmake_minimum_required(VERSION 3.5)
|
||||
|
||||
function(check_file_hash has_hash hash_is_good)
|
||||
if("${has_hash}" STREQUAL "")
|
||||
message(FATAL_ERROR "has_hash Can't be empty")
|
||||
endif()
|
||||
|
||||
if("${hash_is_good}" STREQUAL "")
|
||||
message(FATAL_ERROR "hash_is_good Can't be empty")
|
||||
endif()
|
||||
|
||||
if("@ALGO@" STREQUAL "")
|
||||
# No check
|
||||
set("${has_hash}" FALSE PARENT_SCOPE)
|
||||
set("${hash_is_good}" FALSE PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
set("${has_hash}" TRUE PARENT_SCOPE)
|
||||
|
||||
message(STATUS "verifying file...
|
||||
file='@LOCAL@'")
|
||||
|
||||
file("@ALGO@" "@LOCAL@" actual_value)
|
||||
|
||||
if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
|
||||
set("${hash_is_good}" FALSE PARENT_SCOPE)
|
||||
message(STATUS "@ALGO@ hash of
|
||||
@LOCAL@
|
||||
does not match expected value
|
||||
expected: '@EXPECT_VALUE@'
|
||||
actual: '${actual_value}'")
|
||||
else()
|
||||
set("${hash_is_good}" TRUE PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(sleep_before_download attempt)
|
||||
if(attempt EQUAL 0)
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(attempt EQUAL 1)
|
||||
message(STATUS "Retrying...")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(sleep_seconds 0)
|
||||
|
||||
if(attempt EQUAL 2)
|
||||
set(sleep_seconds 5)
|
||||
elseif(attempt EQUAL 3)
|
||||
set(sleep_seconds 5)
|
||||
elseif(attempt EQUAL 4)
|
||||
set(sleep_seconds 15)
|
||||
elseif(attempt EQUAL 5)
|
||||
set(sleep_seconds 60)
|
||||
elseif(attempt EQUAL 6)
|
||||
set(sleep_seconds 90)
|
||||
elseif(attempt EQUAL 7)
|
||||
set(sleep_seconds 300)
|
||||
else()
|
||||
set(sleep_seconds 1200)
|
||||
endif()
|
||||
|
||||
message(STATUS "Retry after ${sleep_seconds} seconds (attempt #${attempt}) ...")
|
||||
|
||||
execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${sleep_seconds}")
|
||||
endfunction()
|
||||
|
||||
if("@LOCAL@" STREQUAL "")
|
||||
message(FATAL_ERROR "LOCAL can't be empty")
|
||||
endif()
|
||||
|
||||
if("@REMOTE@" STREQUAL "")
|
||||
message(FATAL_ERROR "REMOTE can't be empty")
|
||||
endif()
|
||||
|
||||
if(EXISTS "@LOCAL@")
|
||||
check_file_hash(has_hash hash_is_good)
|
||||
if(has_hash)
|
||||
if(hash_is_good)
|
||||
message(STATUS "File already exists and hash match (skip download):
|
||||
file='@LOCAL@'
|
||||
@ALGO@='@EXPECT_VALUE@'"
|
||||
)
|
||||
return()
|
||||
else()
|
||||
message(STATUS "File already exists but hash mismatch. Removing...")
|
||||
file(REMOVE "@LOCAL@")
|
||||
endif()
|
||||
else()
|
||||
message(STATUS "File already exists but no hash specified (use URL_HASH):
|
||||
file='@LOCAL@'
|
||||
Old file will be removed and new file downloaded from URL."
|
||||
)
|
||||
file(REMOVE "@LOCAL@")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(retry_number 5)
|
||||
|
||||
foreach(i RANGE ${retry_number})
|
||||
sleep_before_download(${i})
|
||||
|
||||
message(STATUS "downloading...
|
||||
src='@REMOTE@'
|
||||
dst='@LOCAL@'
|
||||
timeout='@TIMEOUT_MSG@'")
|
||||
|
||||
@TLS_VERIFY_CODE@
|
||||
@TLS_CAINFO_CODE@
|
||||
|
||||
file(
|
||||
DOWNLOAD
|
||||
"@REMOTE@" "@LOCAL@"
|
||||
@SHOW_PROGRESS@
|
||||
@TIMEOUT_ARGS@
|
||||
STATUS status
|
||||
LOG log
|
||||
)
|
||||
|
||||
list(GET status 0 status_code)
|
||||
list(GET status 1 status_string)
|
||||
|
||||
if(status_code EQUAL 0)
|
||||
check_file_hash(has_hash hash_is_good)
|
||||
if(has_hash AND NOT hash_is_good)
|
||||
message(STATUS "Hash mismatch, removing...")
|
||||
file(REMOVE "@LOCAL@")
|
||||
else()
|
||||
message(STATUS "Downloading... done")
|
||||
return()
|
||||
endif()
|
||||
else()
|
||||
message("error: downloading '@REMOTE@' failed
|
||||
status_code: ${status_code}
|
||||
status_string: ${status_string}
|
||||
log:
|
||||
--- LOG BEGIN ---
|
||||
${log}
|
||||
--- LOG END ---"
|
||||
)
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
message(FATAL_ERROR "Downloading failed")
|
||||
|
|
@ -867,16 +867,11 @@ function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_p
|
|||
endif()
|
||||
|
||||
if("${hash}" MATCHES "${_ep_hash_regex}")
|
||||
string(CONCAT hash_check
|
||||
"if(EXISTS \"${LOCAL}\")\n"
|
||||
" file(\"${CMAKE_MATCH_1}\" \"${LOCAL}\" hash_value)\n"
|
||||
" if(\"x\${hash_value}\" STREQUAL \"x${CMAKE_MATCH_2}\")\n"
|
||||
" return()\n"
|
||||
" endif()\n"
|
||||
"endif()\n"
|
||||
)
|
||||
set(ALGO "${CMAKE_MATCH_1}")
|
||||
set(EXPECT_VALUE "${CMAKE_MATCH_2}")
|
||||
else()
|
||||
set(hash_check "")
|
||||
set(ALGO "")
|
||||
set(EXPECT_VALUE "")
|
||||
endif()
|
||||
|
||||
set(TLS_VERIFY_CODE "")
|
||||
|
|
@ -904,41 +899,23 @@ function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_p
|
|||
set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")")
|
||||
endif()
|
||||
|
||||
file(WRITE ${script_filename}
|
||||
"${hash_check}message(STATUS \"downloading...
|
||||
src='${REMOTE}'
|
||||
dst='${LOCAL}'
|
||||
timeout='${TIMEOUT_MSG}'\")
|
||||
|
||||
${TLS_VERIFY_CODE}
|
||||
${TLS_CAINFO_CODE}
|
||||
|
||||
file(DOWNLOAD
|
||||
\"${REMOTE}\"
|
||||
\"${LOCAL}\"
|
||||
${SHOW_PROGRESS}
|
||||
${TIMEOUT_ARGS}
|
||||
STATUS status
|
||||
LOG log)
|
||||
|
||||
list(GET status 0 status_code)
|
||||
list(GET status 1 status_string)
|
||||
|
||||
if(NOT status_code EQUAL 0)
|
||||
message(FATAL_ERROR \"error: downloading '${REMOTE}' failed
|
||||
status_code: \${status_code}
|
||||
status_string: \${status_string}
|
||||
log: \${log}
|
||||
\")
|
||||
endif()
|
||||
|
||||
message(STATUS \"downloading... done\")
|
||||
"
|
||||
)
|
||||
|
||||
# Used variables:
|
||||
# * TLS_VERIFY_CODE
|
||||
# * TLS_CAINFO_CODE
|
||||
# * ALGO
|
||||
# * EXPECT_VALUE
|
||||
# * REMOTE
|
||||
# * LOCAL
|
||||
# * SHOW_PROGRESS
|
||||
# * TIMEOUT_ARGS
|
||||
# * TIMEOUT_MSG
|
||||
configure_file(
|
||||
"${_ExternalProject_SELF_DIR}/ExternalProject-download.cmake.in"
|
||||
"${script_filename}"
|
||||
@ONLY
|
||||
)
|
||||
endfunction()
|
||||
|
||||
|
||||
function(_ep_write_verifyfile_script script_filename LOCAL hash)
|
||||
if("${hash}" MATCHES "${_ep_hash_regex}")
|
||||
set(ALGO "${CMAKE_MATCH_1}")
|
||||
|
|
|
|||
Loading…
Reference in New Issue