From 89113e125d95fc24eaaec59edaa00b6d428111f5 Mon Sep 17 00:00:00 2001 From: Ruslan Baratov Date: Thu, 19 May 2016 16:35:01 +0300 Subject: [PATCH] ExternalProject: Re-implement download logic as a dedicated script Move the content to a `ExternalProject-download.cmake.in` file and use `configure_file` to generate the final script. Retry logic was not working before because similar script trigger FATAL_ERROR if 'file(DOWNLOAD ...)' exits with nonzero 'status_code'. FATAL_ERROR makes the whole chain of commands stop and '_ep_write_verifyfile_script' retry logic was not used in fact. Default retry number set to 5 with pauses 0, 5, 5, 15, 60 seconds. Some space left for future improvements if needed (90, 300, 1200=20min). Can be controlled by user. --- Modules/ExternalProject-download.cmake.in | 161 ++++++++++++++++++++++ Modules/ExternalProject.cmake | 61 +++----- 2 files changed, 180 insertions(+), 42 deletions(-) create mode 100644 Modules/ExternalProject-download.cmake.in diff --git a/Modules/ExternalProject-download.cmake.in b/Modules/ExternalProject-download.cmake.in new file mode 100644 index 000000000..5b73cd83c --- /dev/null +++ b/Modules/ExternalProject-download.cmake.in @@ -0,0 +1,161 @@ +#============================================================================= +# Copyright 2008-2013 Kitware, Inc. +# Copyright 2016 Ruslan Baratov +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +cmake_minimum_required(VERSION 3.5) + +function(check_file_hash has_hash hash_is_good) + if("${has_hash}" STREQUAL "") + message(FATAL_ERROR "has_hash Can't be empty") + endif() + + if("${hash_is_good}" STREQUAL "") + message(FATAL_ERROR "hash_is_good Can't be empty") + endif() + + if("@ALGO@" STREQUAL "") + # No check + set("${has_hash}" FALSE PARENT_SCOPE) + set("${hash_is_good}" FALSE PARENT_SCOPE) + return() + endif() + + set("${has_hash}" TRUE PARENT_SCOPE) + + message(STATUS "verifying file... + file='@LOCAL@'") + + file("@ALGO@" "@LOCAL@" actual_value) + + if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@") + set("${hash_is_good}" FALSE PARENT_SCOPE) + message(STATUS "@ALGO@ hash of + @LOCAL@ + does not match expected value + expected: '@EXPECT_VALUE@' + actual: '${actual_value}'") + else() + set("${hash_is_good}" TRUE PARENT_SCOPE) + endif() +endfunction() + +function(sleep_before_download attempt) + if(attempt EQUAL 0) + return() + endif() + + if(attempt EQUAL 1) + message(STATUS "Retrying...") + return() + endif() + + set(sleep_seconds 0) + + if(attempt EQUAL 2) + set(sleep_seconds 5) + elseif(attempt EQUAL 3) + set(sleep_seconds 5) + elseif(attempt EQUAL 4) + set(sleep_seconds 15) + elseif(attempt EQUAL 5) + set(sleep_seconds 60) + elseif(attempt EQUAL 6) + set(sleep_seconds 90) + elseif(attempt EQUAL 7) + set(sleep_seconds 300) + else() + set(sleep_seconds 1200) + endif() + + message(STATUS "Retry after ${sleep_seconds} seconds (attempt #${attempt}) ...") + + execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${sleep_seconds}") +endfunction() + +if("@LOCAL@" STREQUAL "") + message(FATAL_ERROR "LOCAL can't be empty") +endif() + +if("@REMOTE@" STREQUAL "") + message(FATAL_ERROR "REMOTE can't be empty") +endif() + +if(EXISTS "@LOCAL@") + check_file_hash(has_hash hash_is_good) + if(has_hash) + if(hash_is_good) + message(STATUS "File already exists and hash match (skip download): + file='@LOCAL@' + @ALGO@='@EXPECT_VALUE@'" + ) + return() + else() + message(STATUS "File already exists but hash mismatch. Removing...") + file(REMOVE "@LOCAL@") + endif() + else() + message(STATUS "File already exists but no hash specified (use URL_HASH): + file='@LOCAL@' +Old file will be removed and new file downloaded from URL." + ) + file(REMOVE "@LOCAL@") + endif() +endif() + +set(retry_number 5) + +foreach(i RANGE ${retry_number}) + sleep_before_download(${i}) + + message(STATUS "downloading... + src='@REMOTE@' + dst='@LOCAL@' + timeout='@TIMEOUT_MSG@'") + + @TLS_VERIFY_CODE@ + @TLS_CAINFO_CODE@ + + file( + DOWNLOAD + "@REMOTE@" "@LOCAL@" + @SHOW_PROGRESS@ + @TIMEOUT_ARGS@ + STATUS status + LOG log + ) + + list(GET status 0 status_code) + list(GET status 1 status_string) + + if(status_code EQUAL 0) + check_file_hash(has_hash hash_is_good) + if(has_hash AND NOT hash_is_good) + message(STATUS "Hash mismatch, removing...") + file(REMOVE "@LOCAL@") + else() + message(STATUS "Downloading... done") + return() + endif() + else() + message("error: downloading '@REMOTE@' failed + status_code: ${status_code} + status_string: ${status_string} + log: + --- LOG BEGIN --- + ${log} + --- LOG END ---" + ) + endif() +endforeach() + +message(FATAL_ERROR "Downloading failed") diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 224950108..ad6de189b 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -867,16 +867,11 @@ function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_p endif() if("${hash}" MATCHES "${_ep_hash_regex}") - string(CONCAT hash_check - "if(EXISTS \"${LOCAL}\")\n" - " file(\"${CMAKE_MATCH_1}\" \"${LOCAL}\" hash_value)\n" - " if(\"x\${hash_value}\" STREQUAL \"x${CMAKE_MATCH_2}\")\n" - " return()\n" - " endif()\n" - "endif()\n" - ) + set(ALGO "${CMAKE_MATCH_1}") + set(EXPECT_VALUE "${CMAKE_MATCH_2}") else() - set(hash_check "") + set(ALGO "") + set(EXPECT_VALUE "") endif() set(TLS_VERIFY_CODE "") @@ -904,41 +899,23 @@ function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_p set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")") endif() - file(WRITE ${script_filename} -"${hash_check}message(STATUS \"downloading... - src='${REMOTE}' - dst='${LOCAL}' - timeout='${TIMEOUT_MSG}'\") - -${TLS_VERIFY_CODE} -${TLS_CAINFO_CODE} - -file(DOWNLOAD - \"${REMOTE}\" - \"${LOCAL}\" - ${SHOW_PROGRESS} - ${TIMEOUT_ARGS} - STATUS status - LOG log) - -list(GET status 0 status_code) -list(GET status 1 status_string) - -if(NOT status_code EQUAL 0) - message(FATAL_ERROR \"error: downloading '${REMOTE}' failed - status_code: \${status_code} - status_string: \${status_string} - log: \${log} -\") -endif() - -message(STATUS \"downloading... done\") -" -) - + # Used variables: + # * TLS_VERIFY_CODE + # * TLS_CAINFO_CODE + # * ALGO + # * EXPECT_VALUE + # * REMOTE + # * LOCAL + # * SHOW_PROGRESS + # * TIMEOUT_ARGS + # * TIMEOUT_MSG + configure_file( + "${_ExternalProject_SELF_DIR}/ExternalProject-download.cmake.in" + "${script_filename}" + @ONLY + ) endfunction() - function(_ep_write_verifyfile_script script_filename LOCAL hash) if("${hash}" MATCHES "${_ep_hash_regex}") set(ALGO "${CMAKE_MATCH_1}")