ExternalData: Add support for custom algorithm-to-URL mapping

Allow URL templates to contain a %(algo:<key>) placeholder that is
replaced by mapping the canonical hash algorithm name through a map
defined by the <key>.

Extend the Module.ExternalData test to cover the behavior.
Extend the RunCMake.ExternalData test to cover error cases.

Help/release/dev/ExternalData-url-algo-map.rst

@@ -0,0 +1,8 @@
+ExternalData-url-algo-map
+-------------------------
+
+* The :module:`ExternalData` module learned a new URL template
+  placeholder ``%(algo:<key>)`` to allow custom mapping from
+  algorithm name to URL component through configuration of new
+  :variable:`ExternalData_URL_ALGO_<algo>_<key>` variables.
+  This allows more flexibility in remote URLs.

Modules/ExternalData.cmake

@@ -155,13 +155,23 @@ calling any of the functions provided by this module.
   inactivity timeout, in seconds, with a default of ``60`` seconds.
   Set to ``0`` to disable enforcement.
 
+.. variable:: ExternalData_URL_ALGO_<algo>_<key>
+
+  Specify a custom URL component to be substituted for URL template
+  placeholders of the form ``%(algo:<key>)``, where ``<key>`` is a
+  valid C identifier, when fetching an object referenced via hash
+  algorithm ``<algo>``.  If not defined, the default URL component
+  is just ``<algo>`` for any ``<key>``.
+
 .. variable:: ExternalData_URL_TEMPLATES
 
   The ``ExternalData_URL_TEMPLATES`` may be set to provide a list of
   of URL templates using the placeholders ``%(algo)`` and ``%(hash)``
   in each template.  Data fetch rules try each URL template in order
   by substituting the hash algorithm name for ``%(algo)`` and the hash
-  value for ``%(hash)``.
+  value for ``%(hash)``.  Alternatively one may use ``%(algo:<key>)``
+  with ``ExternalData_URL_ALGO_<algo>_<key>`` variables to gain more
+  flexibility in remote URLs.
 
 Referencing Files
 ^^^^^^^^^^^^^^^^^
@@ -349,6 +359,25 @@ function(ExternalData_add_target target)
           "The key must be a valid C identifier.")
       endif()
     endif()
+
+    # Store custom algorithm name to URL component maps.
+    if("${url_template}" MATCHES "%\\(algo:([^)]*)\\)")
+      set(key "${CMAKE_MATCH_1}")
+      if(key MATCHES "^[A-Za-z_][A-Za-z0-9_]*$")
+        string(REPLACE "|" ";" _algos "${_ExternalData_REGEX_ALGO}")
+        foreach(algo ${_algos})
+          if(DEFINED ExternalData_URL_ALGO_${algo}_${key})
+            string(CONCAT _ExternalData_CONFIG_CODE "${_ExternalData_CONFIG_CODE}\n"
+              "set(ExternalData_URL_ALGO_${algo}_${key} \"${ExternalData_URL_ALGO_${algo}_${key}}\")")
+          endif()
+        endforeach()
+      else()
+        message(FATAL_ERROR
+          "Bad %(algo:${key}) in URL template:\n"
+          " ${url_template}\n"
+          "The transform name must be a valid C identifier.")
+      endif()
+    endif()
   endforeach()
 
   # Store configuration for use by build-time script.
@@ -904,6 +933,16 @@ function(_ExternalData_download_object name hash algo var_obj)
   foreach(url_template IN LISTS ExternalData_URL_TEMPLATES)
     string(REPLACE "%(hash)" "${hash}" url_tmp "${url_template}")
     string(REPLACE "%(algo)" "${algo}" url "${url_tmp}")
+    if(url MATCHES "^(.*)%\\(algo:([A-Za-z_][A-Za-z0-9_]*)\\)(.*)$")
+      set(lhs "${CMAKE_MATCH_1}")
+      set(key "${CMAKE_MATCH_2}")
+      set(rhs "${CMAKE_MATCH_3}")
+      if(DEFINED ExternalData_URL_ALGO_${algo}_${key})
+        set(url "${lhs}${ExternalData_URL_ALGO_${algo}_${key}}${rhs}")
+      else()
+        set(url "${lhs}${algo}${rhs}")
+      endif()
+    endif()
     message(STATUS "Fetching \"${url}\"")
     if(url MATCHES "^ExternalDataCustomScript://([A-Za-z_][A-Za-z0-9_]*)/(.*)$")
       _ExternalData_custom_fetch("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" "${tmp}" err errMsg)

Tests/Module/ExternalData/Alt/MyAlgoMap1-md5/dded55e43cd6529ee35d24113dfc87a3

@@ -0,0 +1 @@
+DataAlgoMap

Tests/Module/ExternalData/Alt/SHA1/85158f0c1996837976e858c42a9a7634bfe91b93

@@ -0,0 +1 @@
+DataAlgoMap

Tests/Module/ExternalData/CMakeLists.txt

@@ -10,8 +10,10 @@ if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "^/")
 endif()
 set(ExternalData_URL_TEMPLATES
   "file://${slash}${CMAKE_CURRENT_SOURCE_DIR}/%(algo)/%(hash)"
+  "file://${slash}${CMAKE_CURRENT_SOURCE_DIR}/Alt/%(algo:MyAlgoMap1)/%(hash)"
   "ExternalDataCustomScript://MyScript1/%(algo)/%(hash)"
   )
+set(ExternalData_URL_ALGO_MD5_MyAlgoMap1 MyAlgoMap1-md5)
 set(ExternalData_CUSTOM_SCRIPT_MyScript1 "${CMAKE_CURRENT_SOURCE_DIR}/MyScript1.cmake")
 set(ExternalData_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ExternalData")
 file(REMOVE_RECURSE ${ExternalData_BINARY_ROOT}) # clean test
@@ -26,6 +28,8 @@ ExternalData_Add_Test(Data1
     -D Data=DATA{Data.dat}
     ${Data1CheckSpaces}
     -D DataScript=DATA{DataScript.dat}
+    -D DataAlgoMapA=DATA{DataAlgoMapA.dat}
+    -D DataAlgoMapB=DATA{DataAlgoMapB.dat}
     -D DataMissing=DATA{DataMissing.dat}
     -D DataMissingWithAssociated=DATA{DataMissing.dat,Data.dat}
     -D SeriesA=DATA{SeriesA.dat,:}

Tests/Module/ExternalData/Data1Check.cmake

@@ -12,6 +12,14 @@ file(STRINGS "${DataScript}" lines LIMIT_INPUT 1024)
 if(NOT "x${lines}" STREQUAL "xDataScript")
   message(SEND_ERROR "Input file:\n  ${DataScript}\ndoes not have expected content, but [[${lines}]]")
 endif()
+file(STRINGS "${DataAlgoMapA}" lines LIMIT_INPUT 1024)
+if(NOT "x${lines}" STREQUAL "xDataAlgoMap")
+  message(SEND_ERROR "Input file:\n  ${DataAlgoMapA}\ndoes not have expected content, but [[${lines}]]")
+endif()
+file(STRINGS "${DataAlgoMapB}" lines LIMIT_INPUT 1024)
+if(NOT "x${lines}" STREQUAL "xDataAlgoMap")
+  message(SEND_ERROR "Input file:\n  ${DataAlgoMapB}\ndoes not have expected content, but [[${lines}]]")
+endif()
 if(DataMissing)
   if(EXISTS "${DataMissing}")
     message(SEND_ERROR

Tests/Module/ExternalData/DataAlgoMapA.dat.md5

@@ -0,0 +1 @@
+dded55e43cd6529ee35d24113dfc87a3

Tests/Module/ExternalData/DataAlgoMapB.dat.sha1

@@ -0,0 +1 @@
+85158f0c1996837976e858c42a9a7634bfe91b93

Tests/RunCMake/ExternalData/BadAlgoMap1-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadAlgoMap1-stderr.txt

@@ -0,0 +1,9 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Bad %\(algo:\) in URL template:
+
+   file:///path/to/%\(algo:\)/%\(hash\)
+
+  The transform name must be a valid C identifier.
+Call Stack \(most recent call first\):
+  BadAlgoMap1.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/BadAlgoMap1.cmake

@@ -0,0 +1,5 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "file:///path/to/%(algo:)/%(hash)"
+  )
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/BadAlgoMap2-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadAlgoMap2-stderr.txt

@@ -0,0 +1,9 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Bad %\(algo:0BadMap\(\) in URL template:
+
+   file:///path/to/%\(algo:0BadMap\(\)/%\(hash\)
+
+  The transform name must be a valid C identifier.
+Call Stack \(most recent call first\):
+  BadAlgoMap2.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/BadAlgoMap2.cmake

@@ -0,0 +1,5 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "file:///path/to/%(algo:0BadMap()/%(hash)"
+  )
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/RunCMakeTest.cmake

@@ -1,5 +1,7 @@
 include(RunCMake)
 
+run_cmake(BadAlgoMap1)
+run_cmake(BadAlgoMap2)
 run_cmake(BadCustom1)
 run_cmake(BadCustom2)
 run_cmake(BadCustom3)