ExternalData: Allow DATA{} syntax to reference directories

Use a trailing slash to reference a directory.  Require that a list
of associated files be specified to select from within the directory.
One may simply use DATA{Dir/,REGEX:.*} to reference all files but
get a directory passed on the command line.

Modules/ExternalData.cmake

@@ -97,6 +97,14 @@
 # will pass MyInput.mha and MyFrames00.png on the command line but ensure
 # that the associated files are present next to them.
 #
+# The DATA{} syntax may reference a directory using a trailing slash and a
+# list of associated files.  The form DATA{<name>/,<opt1>,<opt2>,...} adds
+# rules to fetch any files in the directory that match one of the associated
+# file options.  For example, the argument DATA{MyDataDir/,REGEX:.*} will pass
+# the full path to a MyDataDir directory on the command line and ensure that
+# the directory contains files corresponding to every file or content link in
+# the MyDataDir source directory.
+#
 # The variable ExternalData_LINK_CONTENT may be set to the name of a supported
 # hash algorithm to enable automatic conversion of real data files referenced
 # by the DATA{} syntax into content links.  For each such <file> a content
@@ -312,11 +320,11 @@ function(_ExternalData_arg target arg options var_file)
   list(GET options 0 data)
   list(REMOVE_AT options 0)
 
-  # Reject trailing slashes.
-  if("x${data}" MATCHES "[/\\]$")
-    message(FATAL_ERROR "Data file reference in argument\n"
-      "  ${arg}\n"
-      "may not end in a slash!")
+  # Interpret trailing slashes as directories.
+  set(data_is_directory 0)
+  if("x${data}" MATCHES "^x(.*)([/\\])$")
+    set(data_is_directory 1)
+    set(data "${CMAKE_MATCH_1}")
   endif()
 
   # Convert to full path.
@@ -338,6 +346,13 @@ function(_ExternalData_arg target arg options var_file)
       "does not lie under the top-level source directory\n"
       "  ${top_src}\n")
   endif()
+  if(data_is_directory AND NOT IS_DIRECTORY "${top_src}/${reldata}")
+    message(FATAL_ERROR "Data directory referenced by argument\n"
+      "  ${arg}\n"
+      "corresponds to source tree path\n"
+      "  ${reldata}\n"
+      "that does not exist as a directory!")
+  endif()
   if(NOT ExternalData_BINARY_ROOT)
     set(ExternalData_BINARY_ROOT "${CMAKE_BINARY_DIR}")
   endif()
@@ -354,7 +369,7 @@ function(_ExternalData_arg target arg options var_file)
 
   set(external "") # Entries external to the source tree.
   set(internal "") # Entries internal to the source tree.
-  set(have_original 0)
+  set(have_original ${data_is_directory})
 
   # Process options.
   set(series_option "")
@@ -378,11 +393,23 @@ function(_ExternalData_arg target arg options var_file)
   endforeach()
 
   if(series_option)
+    if(data_is_directory)
+      message(FATAL_ERROR "Series option \"${series_option}\" not allowed with directories.")
+    endif()
     if(associated_files OR associated_regex)
       message(FATAL_ERROR "Series option \"${series_option}\" not allowed with associated files.")
     endif()
     # Load a whole file series.
     _ExternalData_arg_series()
+  elseif(data_is_directory)
+    if(associated_files OR associated_regex)
+      # Load listed/matching associated files in the directory.
+      _ExternalData_arg_associated()
+    else()
+      message(FATAL_ERROR "Data directory referenced by argument\n"
+        "  ${arg}\n"
+        "must list associated files.")
+    endif()
   else()
     # Load the named data file.
     _ExternalData_arg_single()
@@ -415,7 +442,11 @@ endfunction()
 
 macro(_ExternalData_arg_associated)
   # Associated files lie in the same directory.
+  if(data_is_directory)
+    set(reldir "${reldata}")
+  else()
     get_filename_component(reldir "${reldata}" PATH)
+  endif()
   if(reldir)
     set(reldir "${reldir}/")
   endif()

Tests/Module/ExternalData/CMakeLists.txt

@@ -24,6 +24,7 @@ ExternalData_Add_Test(Data1
     -D SeriesDn=DATA{SeriesDn-1.dat,:}
     -D Paired=DATA{PairedA.dat,PairedB.dat}
     -D Meta=DATA{MetaTop.dat,REGEX:Meta[ABC].dat}
+    -D Directory=DATA{Directory/,A.dat,REGEX:[BC].dat}
     -P ${CMAKE_CURRENT_SOURCE_DIR}/Data1Check.cmake
   )
 ExternalData_Add_Target(Data1)

Tests/Module/ExternalData/Data1Check.cmake

@@ -38,3 +38,9 @@ foreach(n Top A B C)
     message(SEND_ERROR "Input file:\n  ${file}\ndoes not exist!")
   endif()
 endforeach()
+foreach(n A B C)
+  set(file "${Directory}/${n}.dat")
+  if(NOT EXISTS "${file}")
+    message(SEND_ERROR "Input file:\n  ${file}\ndoes not exist!")
+  endif()
+endforeach()

Tests/Module/ExternalData/Directory/A.dat.md5

@@ -0,0 +1 @@
+9d980b06c2f0fec3d4872d68175b9822

Tests/Module/ExternalData/Directory/B.dat.md5

@@ -0,0 +1 @@
+8f4add4581551facf27237e6577fd662

Tests/Module/ExternalData/Directory/C.dat.md5

@@ -0,0 +1 @@
+c1030719c95f3435d8abc39c0d442946

Tests/RunCMake/ExternalData/Directory2-stderr.txt

@@ -1,9 +1,9 @@
 CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
-  Data file reference in argument
+  Data directory referenced by argument
 
     DATA{Directory2/}
 
-  may not end in a slash!
+  must list associated files.
 Call Stack \(most recent call first\):
   .*
   Directory2.cmake:3 \(ExternalData_Add_Test\)

Tests/RunCMake/ExternalData/Directory4-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/Directory4-stderr.txt

@@ -0,0 +1,6 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Series option ":" not allowed with directories.
+Call Stack \(most recent call first\):
+  .*
+  Directory4.cmake:3 \(ExternalData_Add_Test\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/Directory4.cmake

@@ -0,0 +1,6 @@
+include(CTest)
+include(ExternalData)
+ExternalData_Add_Test(Data
+  NAME Test
+  COMMAND ${CMAKE_COMMAND} -E echo DATA{Directory4/,:}
+  )

Tests/RunCMake/ExternalData/Directory5-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/Directory5-stderr.txt

@@ -0,0 +1,14 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Data directory referenced by argument
+
+    DATA{Directory5/}
+
+  corresponds to source tree path
+
+    Directory5
+
+  that does not exist as a directory!
+Call Stack \(most recent call first\):
+  .*
+  Directory5.cmake:3 \(ExternalData_Add_Test\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/Directory5.cmake

@@ -0,0 +1,6 @@
+include(CTest)
+include(ExternalData)
+ExternalData_Add_Test(Data
+  NAME Test
+  COMMAND ${CMAKE_COMMAND} -E echo DATA{Directory5/}
+  )

Tests/RunCMake/ExternalData/RunCMakeTest.cmake

@@ -9,6 +9,8 @@ run_cmake(BadSeries3)
 run_cmake(Directory1)
 run_cmake(Directory2)
 run_cmake(Directory3)
+run_cmake(Directory4)
+run_cmake(Directory5)
 run_cmake(LinkContentMD5)
 run_cmake(MissingData)
 run_cmake(NoLinkInSource)