From 4ab5c652b663377afcdcaf7cda05ac8746b7d21d Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 12 Jan 2015 13:53:33 -0500 Subject: [PATCH 1/7] ExternalData: Convert docs to a bracket comment Use a bracket comment to hold the documentation instead of a block of line comments. This will make further updates easier. --- Modules/ExternalData.cmake | 369 +++++++++++++++++++------------------ 1 file changed, 185 insertions(+), 184 deletions(-) diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 79bb06468..b18674fc9 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -1,189 +1,190 @@ -#.rst: -# ExternalData -# ------------ -# -# Manage data files stored outside source tree -# -# Use this module to unambiguously reference data files stored outside -# the source tree and fetch them at build time from arbitrary local and -# remote content-addressed locations. Functions provided by this module -# recognize arguments with the syntax ``DATA{}`` as references to -# external data, replace them with full paths to local copies of those -# data, and create build rules to fetch and update the local copies. -# -# The ``DATA{}`` syntax is literal and the ```` is a full or relative path -# within the source tree. The source tree must contain either a real -# data file at ```` or a "content link" at ```` containing a -# hash of the real file using a hash algorithm corresponding to ````. -# For example, the argument ``DATA{img.png}`` may be satisfied by either a -# real ``img.png`` file in the current source directory or a ``img.png.md5`` -# file containing its MD5 sum. -# -# The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}`` -# references in its arguments and constructs a new list of arguments: -# -# .. code-block:: cmake -# -# ExternalData_Expand_Arguments( -# # Name of data management target -# # Output variable -# [args...] # Input arguments, DATA{} allowed -# ) -# -# It replaces each ``DATA{}`` reference in an argument with the full path of -# a real data file on disk that will exist after the ```` builds. -# -# The ``ExternalData_Add_Test`` function wraps around the CMake -# :command:`add_test` command but supports ``DATA{}`` references in -# its arguments: -# -# .. code-block:: cmake -# -# ExternalData_Add_Test( -# # Name of data management target -# ... # Arguments of add_test(), DATA{} allowed -# ) -# -# It passes its arguments through ``ExternalData_Expand_Arguments`` and then -# invokes the :command:`add_test` command using the results. -# -# The ``ExternalData_Add_Target`` function creates a custom target to -# manage local instances of data files stored externally: -# -# .. code-block:: cmake -# -# ExternalData_Add_Target( -# # Name of data management target -# ) -# -# It creates custom commands in the target as necessary to make data -# files available for each ``DATA{}`` reference previously evaluated by -# other functions provided by this module. A list of URL templates may -# be provided in the variable ``ExternalData_URL_TEMPLATES`` using the -# placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch -# rules try each URL template in order by substituting the hash -# algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. -# -# The following hash algorithms are supported:: -# -# %(algo) Description -# ------- ----- ----------- -# MD5 .md5 Message-Digest Algorithm 5, RFC 1321 -# SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174 -# SHA224 .sha224 US Secure Hash Algorithms, RFC 4634 -# SHA256 .sha256 US Secure Hash Algorithms, RFC 4634 -# SHA384 .sha384 US Secure Hash Algorithms, RFC 4634 -# SHA512 .sha512 US Secure Hash Algorithms, RFC 4634 -# -# Note that the hashes are used only for unique data identification and -# download verification. -# -# Example usage: -# -# .. code-block:: cmake -# -# include(ExternalData) -# set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)" -# "file:////host/share/%(algo)/%(hash)" -# "http://data.org/%(algo)/%(hash)") -# ExternalData_Add_Test(MyData -# NAME MyTest -# COMMAND MyExe DATA{MyInput.png} -# ) -# ExternalData_Add_Target(MyData) -# -# When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be -# replaced by the full path to a real instance of the data file -# ``MyInput.png`` on disk. If the source tree contains a content link -# such as ``MyInput.png.md5`` then the ``MyData`` target creates a real -# ``MyInput.png`` in the build tree. -# -# The ``DATA{}`` syntax can be told to fetch a file series using the form -# ``DATA{,:}``, where the ``:`` is literal. If the source tree -# contains a group of files or content links named like a series then a -# reference to one member adds rules to fetch all of them. Although all -# members of a series are fetched, only the file originally named by the -# ``DATA{}`` argument is substituted for it. The default configuration -# recognizes file series names ending with ``#.ext``, ``_#.ext``, ``.#.ext``, -# or ``-#.ext`` where ``#`` is a sequence of decimal digits and ``.ext`` is -# any single extension. Configure it with a regex that parses ```` -# and ```` parts from the end of ````:: -# -# ExternalData_SERIES_PARSE = regex of the form ()()$ -# -# For more complicated cases set:: -# -# ExternalData_SERIES_PARSE = regex with at least two () groups -# ExternalData_SERIES_PARSE_PREFIX = regex group number, if any -# ExternalData_SERIES_PARSE_NUMBER = regex group number -# ExternalData_SERIES_PARSE_SUFFIX = regex group number -# -# Configure series number matching with a regex that matches the -# ```` part of series members named ````:: -# -# ExternalData_SERIES_MATCH = regex matching in all series members -# -# Note that the ```` of a series does not include a hash-algorithm -# extension. -# -# The ``DATA{}`` syntax can alternatively match files associated with the -# named file and contained in the same directory. Associated files may -# be specified by options using the syntax -# ``DATA{,,,...}``. Each option may specify one file by -# name or specify a regular expression to match file names using the -# syntax ``REGEX:``. For example, the arguments:: -# -# DATA{MyData/MyInput.mhd,MyInput.img} # File pair -# DATA{MyData/MyFrames00.png,REGEX:MyFrames[0-9]+\\.png} # Series -# -# will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but -# ensure that the associated files are present next to them. -# -# The ``DATA{}`` syntax may reference a directory using a trailing slash and -# a list of associated files. The form ``DATA{/,,,...}`` -# adds rules to fetch any files in the directory that match one of the -# associated file options. For example, the argument -# ``DATA{MyDataDir/,REGEX:.*}`` will pass the full path to a ``MyDataDir`` -# directory on the command line and ensure that the directory contains -# files corresponding to every file or content link in the ``MyDataDir`` -# source directory. -# -# The variable ``ExternalData_LINK_CONTENT`` may be set to the name of a -# supported hash algorithm to enable automatic conversion of real data -# files referenced by the ``DATA{}`` syntax into content links. For each -# such ```` a content link named ```` is created. The -# original file is renamed to the form ``.ExternalData__`` to -# stage it for future transmission to one of the locations in the list -# of URL templates (by means outside the scope of this module). The -# data fetch rule created for the content link will use the staged -# object if it cannot be found using any URL template. -# -# The variable ``ExternalData_OBJECT_STORES`` may be set to a list of local -# directories that store objects using the layout ``/%(algo)/%(hash)``. -# These directories will be searched first for a needed object. If the -# object is not available in any store then it will be fetched remotely -# using the URL templates and added to the first local store listed. If -# no stores are specified the default is a location inside the build -# tree. -# -# The variable ``ExternalData_SOURCE_ROOT`` may be set to the highest source -# directory containing any path named by a ``DATA{}`` reference. The -# default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and -# ``CMAKE_SOURCE_DIR`` must refer to directories within a single source -# distribution (e.g. they come together in one tarball). -# -# The variable ``ExternalData_BINARY_ROOT`` may be set to the directory to -# hold the real data files named by expanded ``DATA{}`` references. The -# default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of -# content links under ``ExternalData_SOURCE_ROOT``. -# -# Variables ``ExternalData_TIMEOUT_INACTIVITY`` and -# ``ExternalData_TIMEOUT_ABSOLUTE`` set the download inactivity and absolute -# timeouts, in seconds. The defaults are 60 seconds and 300 seconds, -# respectively. Set either timeout to 0 seconds to disable enforcement. +#[=======================================================================[.rst: +ExternalData +------------ + +Manage data files stored outside source tree + +Use this module to unambiguously reference data files stored outside +the source tree and fetch them at build time from arbitrary local and +remote content-addressed locations. Functions provided by this module +recognize arguments with the syntax ``DATA{}`` as references to +external data, replace them with full paths to local copies of those +data, and create build rules to fetch and update the local copies. + +The ``DATA{}`` syntax is literal and the ```` is a full or relative path +within the source tree. The source tree must contain either a real +data file at ```` or a "content link" at ```` containing a +hash of the real file using a hash algorithm corresponding to ````. +For example, the argument ``DATA{img.png}`` may be satisfied by either a +real ``img.png`` file in the current source directory or a ``img.png.md5`` +file containing its MD5 sum. + +The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}`` +references in its arguments and constructs a new list of arguments: + +.. code-block:: cmake + + ExternalData_Expand_Arguments( + # Name of data management target + # Output variable + [args...] # Input arguments, DATA{} allowed + ) + +It replaces each ``DATA{}`` reference in an argument with the full path of +a real data file on disk that will exist after the ```` builds. + +The ``ExternalData_Add_Test`` function wraps around the CMake +:command:`add_test` command but supports ``DATA{}`` references in +its arguments: + +.. code-block:: cmake + + ExternalData_Add_Test( + # Name of data management target + ... # Arguments of add_test(), DATA{} allowed + ) + +It passes its arguments through ``ExternalData_Expand_Arguments`` and then +invokes the :command:`add_test` command using the results. + +The ``ExternalData_Add_Target`` function creates a custom target to +manage local instances of data files stored externally: + +.. code-block:: cmake + + ExternalData_Add_Target( + # Name of data management target + ) + +It creates custom commands in the target as necessary to make data +files available for each ``DATA{}`` reference previously evaluated by +other functions provided by this module. A list of URL templates may +be provided in the variable ``ExternalData_URL_TEMPLATES`` using the +placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch +rules try each URL template in order by substituting the hash +algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. + +The following hash algorithms are supported:: + + %(algo) Description + ------- ----- ----------- + MD5 .md5 Message-Digest Algorithm 5, RFC 1321 + SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174 + SHA224 .sha224 US Secure Hash Algorithms, RFC 4634 + SHA256 .sha256 US Secure Hash Algorithms, RFC 4634 + SHA384 .sha384 US Secure Hash Algorithms, RFC 4634 + SHA512 .sha512 US Secure Hash Algorithms, RFC 4634 + +Note that the hashes are used only for unique data identification and +download verification. + +Example usage: + +.. code-block:: cmake + + include(ExternalData) + set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)" + "file:////host/share/%(algo)/%(hash)" + "http://data.org/%(algo)/%(hash)") + ExternalData_Add_Test(MyData + NAME MyTest + COMMAND MyExe DATA{MyInput.png} + ) + ExternalData_Add_Target(MyData) + +When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be +replaced by the full path to a real instance of the data file +``MyInput.png`` on disk. If the source tree contains a content link +such as ``MyInput.png.md5`` then the ``MyData`` target creates a real +``MyInput.png`` in the build tree. + +The ``DATA{}`` syntax can be told to fetch a file series using the form +``DATA{,:}``, where the ``:`` is literal. If the source tree +contains a group of files or content links named like a series then a +reference to one member adds rules to fetch all of them. Although all +members of a series are fetched, only the file originally named by the +``DATA{}`` argument is substituted for it. The default configuration +recognizes file series names ending with ``#.ext``, ``_#.ext``, ``.#.ext``, +or ``-#.ext`` where ``#`` is a sequence of decimal digits and ``.ext`` is +any single extension. Configure it with a regex that parses ```` +and ```` parts from the end of ````:: + + ExternalData_SERIES_PARSE = regex of the form ()()$ + +For more complicated cases set:: + + ExternalData_SERIES_PARSE = regex with at least two () groups + ExternalData_SERIES_PARSE_PREFIX = regex group number, if any + ExternalData_SERIES_PARSE_NUMBER = regex group number + ExternalData_SERIES_PARSE_SUFFIX = regex group number + +Configure series number matching with a regex that matches the +```` part of series members named ````:: + + ExternalData_SERIES_MATCH = regex matching in all series members + +Note that the ```` of a series does not include a hash-algorithm +extension. + +The ``DATA{}`` syntax can alternatively match files associated with the +named file and contained in the same directory. Associated files may +be specified by options using the syntax +``DATA{,,,...}``. Each option may specify one file by +name or specify a regular expression to match file names using the +syntax ``REGEX:``. For example, the arguments:: + + DATA{MyData/MyInput.mhd,MyInput.img} # File pair + DATA{MyData/MyFrames00.png,REGEX:MyFrames[0-9]+\\.png} # Series + +will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but +ensure that the associated files are present next to them. + +The ``DATA{}`` syntax may reference a directory using a trailing slash and +a list of associated files. The form ``DATA{/,,,...}`` +adds rules to fetch any files in the directory that match one of the +associated file options. For example, the argument +``DATA{MyDataDir/,REGEX:.*}`` will pass the full path to a ``MyDataDir`` +directory on the command line and ensure that the directory contains +files corresponding to every file or content link in the ``MyDataDir`` +source directory. + +The variable ``ExternalData_LINK_CONTENT`` may be set to the name of a +supported hash algorithm to enable automatic conversion of real data +files referenced by the ``DATA{}`` syntax into content links. For each +such ```` a content link named ```` is created. The +original file is renamed to the form ``.ExternalData__`` to +stage it for future transmission to one of the locations in the list +of URL templates (by means outside the scope of this module). The +data fetch rule created for the content link will use the staged +object if it cannot be found using any URL template. + +The variable ``ExternalData_OBJECT_STORES`` may be set to a list of local +directories that store objects using the layout ``/%(algo)/%(hash)``. +These directories will be searched first for a needed object. If the +object is not available in any store then it will be fetched remotely +using the URL templates and added to the first local store listed. If +no stores are specified the default is a location inside the build +tree. + +The variable ``ExternalData_SOURCE_ROOT`` may be set to the highest source +directory containing any path named by a ``DATA{}`` reference. The +default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and +``CMAKE_SOURCE_DIR`` must refer to directories within a single source +distribution (e.g. they come together in one tarball). + +The variable ``ExternalData_BINARY_ROOT`` may be set to the directory to +hold the real data files named by expanded ``DATA{}`` references. The +default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of +content links under ``ExternalData_SOURCE_ROOT``. + +Variables ``ExternalData_TIMEOUT_INACTIVITY`` and +``ExternalData_TIMEOUT_ABSOLUTE`` set the download inactivity and absolute +timeouts, in seconds. The defaults are 60 seconds and 300 seconds, +respectively. Set either timeout to 0 seconds to disable enforcement. +#]=======================================================================] #============================================================================= -# Copyright 2010-2013 Kitware, Inc. +# Copyright 2010-2015 Kitware, Inc. # # Distributed under the OSI-approved BSD License (the "License"); # see accompanying file Copyright.txt for details. From f3884b47ec64b80ad31da93192188f05f60ae5d8 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 12 Jan 2015 14:38:22 -0500 Subject: [PATCH 2/7] ExternalData: Split documentation into sections Also explicitly mark functions and variables. --- Modules/ExternalData.cmake | 170 +++++++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 63 deletions(-) diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index b18674fc9..5b01294f9 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -2,8 +2,15 @@ ExternalData ------------ +.. only:: html + + .. contents:: + Manage data files stored outside source tree +Introduction +^^^^^^^^^^^^ + Use this module to unambiguously reference data files stored outside the source tree and fetch them at build time from arbitrary local and remote content-addressed locations. Functions provided by this module @@ -19,50 +26,56 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a real ``img.png`` file in the current source directory or a ``img.png.md5`` file containing its MD5 sum. -The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}`` -references in its arguments and constructs a new list of arguments: +Module Functions +^^^^^^^^^^^^^^^^ -.. code-block:: cmake +.. command:: ExternalData_Expand_Arguments - ExternalData_Expand_Arguments( - # Name of data management target - # Output variable - [args...] # Input arguments, DATA{} allowed - ) + The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}`` + references in its arguments and constructs a new list of arguments:: -It replaces each ``DATA{}`` reference in an argument with the full path of -a real data file on disk that will exist after the ```` builds. + ExternalData_Expand_Arguments( + # Name of data management target + # Output variable + [args...] # Input arguments, DATA{} allowed + ) -The ``ExternalData_Add_Test`` function wraps around the CMake -:command:`add_test` command but supports ``DATA{}`` references in -its arguments: + It replaces each ``DATA{}`` reference in an argument with the full path of + a real data file on disk that will exist after the ```` builds. -.. code-block:: cmake +.. command:: ExternalData_Add_Test - ExternalData_Add_Test( - # Name of data management target - ... # Arguments of add_test(), DATA{} allowed - ) + The ``ExternalData_Add_Test`` function wraps around the CMake + :command:`add_test` command but supports ``DATA{}`` references in + its arguments:: -It passes its arguments through ``ExternalData_Expand_Arguments`` and then -invokes the :command:`add_test` command using the results. + ExternalData_Add_Test( + # Name of data management target + ... # Arguments of add_test(), DATA{} allowed + ) -The ``ExternalData_Add_Target`` function creates a custom target to -manage local instances of data files stored externally: + It passes its arguments through ``ExternalData_Expand_Arguments`` and then + invokes the :command:`add_test` command using the results. -.. code-block:: cmake +.. command:: ExternalData_Add_Target - ExternalData_Add_Target( - # Name of data management target - ) + The ``ExternalData_Add_Target`` function creates a custom target to + manage local instances of data files stored externally:: -It creates custom commands in the target as necessary to make data -files available for each ``DATA{}`` reference previously evaluated by -other functions provided by this module. A list of URL templates may -be provided in the variable ``ExternalData_URL_TEMPLATES`` using the -placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch -rules try each URL template in order by substituting the hash -algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. + ExternalData_Add_Target( + # Name of data management target + ) + + It creates custom commands in the target as necessary to make data + files available for each ``DATA{}`` reference previously evaluated by + other functions provided by this module. A list of URL templates may + be provided in the variable ``ExternalData_URL_TEMPLATES`` using the + placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch + rules try each URL template in order by substituting the hash + algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. + +Hash Algorithms +^^^^^^^^^^^^^^^ The following hash algorithms are supported:: @@ -78,7 +91,8 @@ The following hash algorithms are supported:: Note that the hashes are used only for unique data identification and download verification. -Example usage: +Example Usage +^^^^^^^^^^^^^ .. code-block:: cmake @@ -98,6 +112,9 @@ replaced by the full path to a real instance of the data file such as ``MyInput.png.md5`` then the ``MyData`` target creates a real ``MyInput.png`` in the build tree. +Referencing File Series +^^^^^^^^^^^^^^^^^^^^^^^ + The ``DATA{}`` syntax can be told to fetch a file series using the form ``DATA{,:}``, where the ``:`` is literal. If the source tree contains a group of files or content links named like a series then a @@ -126,6 +143,9 @@ Configure series number matching with a regex that matches the Note that the ```` of a series does not include a hash-algorithm extension. +Referencing Associated Files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + The ``DATA{}`` syntax can alternatively match files associated with the named file and contained in the same directory. Associated files may be specified by options using the syntax @@ -139,6 +159,9 @@ syntax ``REGEX:``. For example, the arguments:: will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but ensure that the associated files are present next to them. +Referencing Directories +^^^^^^^^^^^^^^^^^^^^^^^ + The ``DATA{}`` syntax may reference a directory using a trailing slash and a list of associated files. The form ``DATA{/,,,...}`` adds rules to fetch any files in the directory that match one of the @@ -148,39 +171,60 @@ directory on the command line and ensure that the directory contains files corresponding to every file or content link in the ``MyDataDir`` source directory. -The variable ``ExternalData_LINK_CONTENT`` may be set to the name of a -supported hash algorithm to enable automatic conversion of real data -files referenced by the ``DATA{}`` syntax into content links. For each -such ```` a content link named ```` is created. The -original file is renamed to the form ``.ExternalData__`` to -stage it for future transmission to one of the locations in the list -of URL templates (by means outside the scope of this module). The -data fetch rule created for the content link will use the staged -object if it cannot be found using any URL template. +Module Variables +^^^^^^^^^^^^^^^^ -The variable ``ExternalData_OBJECT_STORES`` may be set to a list of local -directories that store objects using the layout ``/%(algo)/%(hash)``. -These directories will be searched first for a needed object. If the -object is not available in any store then it will be fetched remotely -using the URL templates and added to the first local store listed. If -no stores are specified the default is a location inside the build -tree. +The following variables configure behavior. They should be set before +calling any of the functions provided by this module. -The variable ``ExternalData_SOURCE_ROOT`` may be set to the highest source -directory containing any path named by a ``DATA{}`` reference. The -default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and -``CMAKE_SOURCE_DIR`` must refer to directories within a single source -distribution (e.g. they come together in one tarball). +.. variable:: ExternalData_LINK_CONTENT -The variable ``ExternalData_BINARY_ROOT`` may be set to the directory to -hold the real data files named by expanded ``DATA{}`` references. The -default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of -content links under ``ExternalData_SOURCE_ROOT``. + The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a + supported hash algorithm to enable automatic conversion of real data + files referenced by the ``DATA{}`` syntax into content links. For each + such ```` a content link named ```` is created. The + original file is renamed to the form ``.ExternalData__`` to + stage it for future transmission to one of the locations in the list + of URL templates (by means outside the scope of this module). The + data fetch rule created for the content link will use the staged + object if it cannot be found using any URL template. -Variables ``ExternalData_TIMEOUT_INACTIVITY`` and -``ExternalData_TIMEOUT_ABSOLUTE`` set the download inactivity and absolute -timeouts, in seconds. The defaults are 60 seconds and 300 seconds, -respectively. Set either timeout to 0 seconds to disable enforcement. +.. variable:: ExternalData_OBJECT_STORES + + The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local + directories that store objects using the layout ``/%(algo)/%(hash)``. + These directories will be searched first for a needed object. If the + object is not available in any store then it will be fetched remotely + using the URL templates and added to the first local store listed. If + no stores are specified the default is a location inside the build + tree. + +.. variable:: ExternalData_SOURCE_ROOT + + The ``ExternalData_SOURCE_ROOT`` variable may be set to the highest source + directory containing any path named by a ``DATA{}`` reference. The + default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and + ``CMAKE_SOURCE_DIR`` must refer to directories within a single source + distribution (e.g. they come together in one tarball). + +.. variable:: ExternalData_BINARY_ROOT + + The ``ExternalData_BINARY_ROOT`` variable may be set to the directory to + hold the real data files named by expanded ``DATA{}`` references. The + default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of + content links under ``ExternalData_SOURCE_ROOT``. + +.. variable:: ExternalData_TIMEOUT_INACTIVITY + + The ``ExternalData_TIMEOUT_INACTIVITY`` variable sets the download + inactivity timeout, in seconds, with a default of ``60`` seconds. + Set to ``0`` to disable enforcement. + +.. variable:: ExternalData_TIMEOUT_ABSOLUTE + + The ``ExternalData_TIMEOUT_ABSOLUTE`` variable sets the download + absolute timeout, in seconds, with a default of ``300`` seconds. + Set to ``0`` to disable enforcement. #]=======================================================================] #============================================================================= From 531e75e0b73f921fc02983f19a07141c867a752a Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 12 Jan 2015 14:58:17 -0500 Subject: [PATCH 3/7] ExternalData: Document all variables defined by module Extend the "Module Variables" section to list variables previously mentioned only in text. --- Modules/ExternalData.cmake | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 5b01294f9..20180aeb8 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -68,11 +68,11 @@ Module Functions It creates custom commands in the target as necessary to make data files available for each ``DATA{}`` reference previously evaluated by - other functions provided by this module. A list of URL templates may - be provided in the variable ``ExternalData_URL_TEMPLATES`` using the - placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch - rules try each URL template in order by substituting the hash - algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. + other functions provided by this module. + Data files may be fetched from one of the URL templates specified in + the ``ExternalData_URL_TEMPLATES`` variable, or may be found locally + in one of the paths specified in the ``ExternalData_OBJECT_STORES`` + variable. Hash Algorithms ^^^^^^^^^^^^^^^ @@ -177,6 +177,14 @@ Module Variables The following variables configure behavior. They should be set before calling any of the functions provided by this module. +.. variable:: ExternalData_URL_TEMPLATES + + The ``ExternalData_URL_TEMPLATES`` may be set to provide a list of + of URL templates using the placeholders ``%(algo)`` and ``%(hash)`` + in each template. Data fetch rules try each URL template in order + by substituting the hash algorithm name for ``%(algo)`` and the hash + value for ``%(hash)``. + .. variable:: ExternalData_LINK_CONTENT The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a @@ -225,6 +233,14 @@ calling any of the functions provided by this module. The ``ExternalData_TIMEOUT_ABSOLUTE`` variable sets the download absolute timeout, in seconds, with a default of ``300`` seconds. Set to ``0`` to disable enforcement. + +.. variable:: ExternalData_SERIES_PARSE + ExternalData_SERIES_PARSE_PREFIX + ExternalData_SERIES_PARSE_NUMBER + ExternalData_SERIES_PARSE_SUFFIX + ExternalData_SERIES_MATCH + + See `Referencing File Series`_. #]=======================================================================] #============================================================================= From a32b2245ca8d77da5b8218bcdcc6958016cdbb48 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 12 Jan 2015 15:10:18 -0500 Subject: [PATCH 4/7] ExternalData: Re-order documentation Put example usage just after the intro. Put hash algos last. Put variables right after functions since they both cover the module API. --- Modules/ExternalData.cmake | 194 ++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 20180aeb8..313d27d6a 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -26,6 +26,27 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a real ``img.png`` file in the current source directory or a ``img.png.md5`` file containing its MD5 sum. +Example Usage +^^^^^^^^^^^^^ + +.. code-block:: cmake + + include(ExternalData) + set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)" + "file:////host/share/%(algo)/%(hash)" + "http://data.org/%(algo)/%(hash)") + ExternalData_Add_Test(MyData + NAME MyTest + COMMAND MyExe DATA{MyInput.png} + ) + ExternalData_Add_Target(MyData) + +When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be +replaced by the full path to a real instance of the data file +``MyInput.png`` on disk. If the source tree contains a content link +such as ``MyInput.png.md5`` then the ``MyData`` target creates a real +``MyInput.png`` in the build tree. + Module Functions ^^^^^^^^^^^^^^^^ @@ -74,43 +95,76 @@ Module Functions in one of the paths specified in the ``ExternalData_OBJECT_STORES`` variable. -Hash Algorithms -^^^^^^^^^^^^^^^ +Module Variables +^^^^^^^^^^^^^^^^ -The following hash algorithms are supported:: +The following variables configure behavior. They should be set before +calling any of the functions provided by this module. - %(algo) Description - ------- ----- ----------- - MD5 .md5 Message-Digest Algorithm 5, RFC 1321 - SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174 - SHA224 .sha224 US Secure Hash Algorithms, RFC 4634 - SHA256 .sha256 US Secure Hash Algorithms, RFC 4634 - SHA384 .sha384 US Secure Hash Algorithms, RFC 4634 - SHA512 .sha512 US Secure Hash Algorithms, RFC 4634 +.. variable:: ExternalData_BINARY_ROOT -Note that the hashes are used only for unique data identification and -download verification. + The ``ExternalData_BINARY_ROOT`` variable may be set to the directory to + hold the real data files named by expanded ``DATA{}`` references. The + default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of + content links under ``ExternalData_SOURCE_ROOT``. -Example Usage -^^^^^^^^^^^^^ +.. variable:: ExternalData_LINK_CONTENT -.. code-block:: cmake + The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a + supported hash algorithm to enable automatic conversion of real data + files referenced by the ``DATA{}`` syntax into content links. For each + such ```` a content link named ```` is created. The + original file is renamed to the form ``.ExternalData__`` to + stage it for future transmission to one of the locations in the list + of URL templates (by means outside the scope of this module). The + data fetch rule created for the content link will use the staged + object if it cannot be found using any URL template. - include(ExternalData) - set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)" - "file:////host/share/%(algo)/%(hash)" - "http://data.org/%(algo)/%(hash)") - ExternalData_Add_Test(MyData - NAME MyTest - COMMAND MyExe DATA{MyInput.png} - ) - ExternalData_Add_Target(MyData) +.. variable:: ExternalData_OBJECT_STORES -When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be -replaced by the full path to a real instance of the data file -``MyInput.png`` on disk. If the source tree contains a content link -such as ``MyInput.png.md5`` then the ``MyData`` target creates a real -``MyInput.png`` in the build tree. + The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local + directories that store objects using the layout ``/%(algo)/%(hash)``. + These directories will be searched first for a needed object. If the + object is not available in any store then it will be fetched remotely + using the URL templates and added to the first local store listed. If + no stores are specified the default is a location inside the build + tree. + +.. variable:: ExternalData_SERIES_PARSE + ExternalData_SERIES_PARSE_PREFIX + ExternalData_SERIES_PARSE_NUMBER + ExternalData_SERIES_PARSE_SUFFIX + ExternalData_SERIES_MATCH + + See `Referencing File Series`_. + +.. variable:: ExternalData_SOURCE_ROOT + + The ``ExternalData_SOURCE_ROOT`` variable may be set to the highest source + directory containing any path named by a ``DATA{}`` reference. The + default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and + ``CMAKE_SOURCE_DIR`` must refer to directories within a single source + distribution (e.g. they come together in one tarball). + +.. variable:: ExternalData_TIMEOUT_ABSOLUTE + + The ``ExternalData_TIMEOUT_ABSOLUTE`` variable sets the download + absolute timeout, in seconds, with a default of ``300`` seconds. + Set to ``0`` to disable enforcement. + +.. variable:: ExternalData_TIMEOUT_INACTIVITY + + The ``ExternalData_TIMEOUT_INACTIVITY`` variable sets the download + inactivity timeout, in seconds, with a default of ``60`` seconds. + Set to ``0`` to disable enforcement. + +.. variable:: ExternalData_URL_TEMPLATES + + The ``ExternalData_URL_TEMPLATES`` may be set to provide a list of + of URL templates using the placeholders ``%(algo)`` and ``%(hash)`` + in each template. Data fetch rules try each URL template in order + by substituting the hash algorithm name for ``%(algo)`` and the hash + value for ``%(hash)``. Referencing File Series ^^^^^^^^^^^^^^^^^^^^^^^ @@ -171,76 +225,22 @@ directory on the command line and ensure that the directory contains files corresponding to every file or content link in the ``MyDataDir`` source directory. -Module Variables -^^^^^^^^^^^^^^^^ +Hash Algorithms +^^^^^^^^^^^^^^^ -The following variables configure behavior. They should be set before -calling any of the functions provided by this module. +The following hash algorithms are supported:: -.. variable:: ExternalData_URL_TEMPLATES + %(algo) Description + ------- ----- ----------- + MD5 .md5 Message-Digest Algorithm 5, RFC 1321 + SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174 + SHA224 .sha224 US Secure Hash Algorithms, RFC 4634 + SHA256 .sha256 US Secure Hash Algorithms, RFC 4634 + SHA384 .sha384 US Secure Hash Algorithms, RFC 4634 + SHA512 .sha512 US Secure Hash Algorithms, RFC 4634 - The ``ExternalData_URL_TEMPLATES`` may be set to provide a list of - of URL templates using the placeholders ``%(algo)`` and ``%(hash)`` - in each template. Data fetch rules try each URL template in order - by substituting the hash algorithm name for ``%(algo)`` and the hash - value for ``%(hash)``. - -.. variable:: ExternalData_LINK_CONTENT - - The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a - supported hash algorithm to enable automatic conversion of real data - files referenced by the ``DATA{}`` syntax into content links. For each - such ```` a content link named ```` is created. The - original file is renamed to the form ``.ExternalData__`` to - stage it for future transmission to one of the locations in the list - of URL templates (by means outside the scope of this module). The - data fetch rule created for the content link will use the staged - object if it cannot be found using any URL template. - -.. variable:: ExternalData_OBJECT_STORES - - The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local - directories that store objects using the layout ``/%(algo)/%(hash)``. - These directories will be searched first for a needed object. If the - object is not available in any store then it will be fetched remotely - using the URL templates and added to the first local store listed. If - no stores are specified the default is a location inside the build - tree. - -.. variable:: ExternalData_SOURCE_ROOT - - The ``ExternalData_SOURCE_ROOT`` variable may be set to the highest source - directory containing any path named by a ``DATA{}`` reference. The - default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and - ``CMAKE_SOURCE_DIR`` must refer to directories within a single source - distribution (e.g. they come together in one tarball). - -.. variable:: ExternalData_BINARY_ROOT - - The ``ExternalData_BINARY_ROOT`` variable may be set to the directory to - hold the real data files named by expanded ``DATA{}`` references. The - default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of - content links under ``ExternalData_SOURCE_ROOT``. - -.. variable:: ExternalData_TIMEOUT_INACTIVITY - - The ``ExternalData_TIMEOUT_INACTIVITY`` variable sets the download - inactivity timeout, in seconds, with a default of ``60`` seconds. - Set to ``0`` to disable enforcement. - -.. variable:: ExternalData_TIMEOUT_ABSOLUTE - - The ``ExternalData_TIMEOUT_ABSOLUTE`` variable sets the download - absolute timeout, in seconds, with a default of ``300`` seconds. - Set to ``0`` to disable enforcement. - -.. variable:: ExternalData_SERIES_PARSE - ExternalData_SERIES_PARSE_PREFIX - ExternalData_SERIES_PARSE_NUMBER - ExternalData_SERIES_PARSE_SUFFIX - ExternalData_SERIES_MATCH - - See `Referencing File Series`_. +Note that the hashes are used only for unique data identification and +download verification. #]=======================================================================] #============================================================================= From 945571db747fd09036d0efd55b538dd4b3c8b874 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 12 Jan 2015 15:14:45 -0500 Subject: [PATCH 5/7] ExternalData: Improve documentation organization Move the basic DATA{} description to a section just before the file series description. Move all sections on referencing files into subsections of a common "Referencing Files" section. Subsume example usage into the introduction since it gives a high-level starting point to understand the rest of the docs. --- Modules/ExternalData.cmake | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 313d27d6a..4f84d53d1 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -18,16 +18,7 @@ recognize arguments with the syntax ``DATA{}`` as references to external data, replace them with full paths to local copies of those data, and create build rules to fetch and update the local copies. -The ``DATA{}`` syntax is literal and the ```` is a full or relative path -within the source tree. The source tree must contain either a real -data file at ```` or a "content link" at ```` containing a -hash of the real file using a hash algorithm corresponding to ````. -For example, the argument ``DATA{img.png}`` may be satisfied by either a -real ``img.png`` file in the current source directory or a ``img.png.md5`` -file containing its MD5 sum. - -Example Usage -^^^^^^^^^^^^^ +For example: .. code-block:: cmake @@ -166,8 +157,22 @@ calling any of the functions provided by this module. by substituting the hash algorithm name for ``%(algo)`` and the hash value for ``%(hash)``. +Referencing Files +^^^^^^^^^^^^^^^^^ + +Referencing Single Files +"""""""""""""""""""""""" + +The ``DATA{}`` syntax is literal and the ```` is a full or relative path +within the source tree. The source tree must contain either a real +data file at ```` or a "content link" at ```` containing a +hash of the real file using a hash algorithm corresponding to ````. +For example, the argument ``DATA{img.png}`` may be satisfied by either a +real ``img.png`` file in the current source directory or a ``img.png.md5`` +file containing its MD5 sum. + Referencing File Series -^^^^^^^^^^^^^^^^^^^^^^^ +""""""""""""""""""""""" The ``DATA{}`` syntax can be told to fetch a file series using the form ``DATA{,:}``, where the ``:`` is literal. If the source tree @@ -198,7 +203,7 @@ Note that the ```` of a series does not include a hash-algorithm extension. Referencing Associated Files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +"""""""""""""""""""""""""""" The ``DATA{}`` syntax can alternatively match files associated with the named file and contained in the same directory. Associated files may @@ -214,7 +219,7 @@ will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but ensure that the associated files are present next to them. Referencing Directories -^^^^^^^^^^^^^^^^^^^^^^^ +""""""""""""""""""""""" The ``DATA{}`` syntax may reference a directory using a trailing slash and a list of associated files. The form ``DATA{/,,,...}`` From 0fe4d8bb3b9e8074f7efd95ae92fa32fe29a2c02 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 13 Jan 2015 10:03:56 -0500 Subject: [PATCH 6/7] ExternalData: Add support for custom download scripts Add support for a special URL template to map the fetch operation to a project-specified .cmake script insead of using file(DOWNLOAD). Extend the Module.ExternalData test to cover the behavior. Extend the RunCMake.ExternalData test to cover error cases. --- Modules/ExternalData.cmake | 105 +++++++++++++++++- Modules/ExternalData_config.cmake.in | 1 + Tests/Module/ExternalData/CMakeLists.txt | 3 + Tests/Module/ExternalData/Data1Check.cmake | 4 + Tests/Module/ExternalData/DataScript.dat.md5 | 1 + Tests/Module/ExternalData/MyScript1.cmake | 5 + .../ExternalData/BadCustom1-result.txt | 1 + .../ExternalData/BadCustom1-stderr.txt | 9 ++ Tests/RunCMake/ExternalData/BadCustom1.cmake | 5 + .../ExternalData/BadCustom2-result.txt | 1 + .../ExternalData/BadCustom2-stderr.txt | 9 ++ Tests/RunCMake/ExternalData/BadCustom2.cmake | 5 + .../ExternalData/BadCustom3-result.txt | 1 + .../ExternalData/BadCustom3-stderr.txt | 7 ++ Tests/RunCMake/ExternalData/BadCustom3.cmake | 5 + .../ExternalData/BadCustom4-result.txt | 1 + .../ExternalData/BadCustom4-stderr.txt | 7 ++ Tests/RunCMake/ExternalData/BadCustom4.cmake | 6 + .../RunCMake/ExternalData/RunCMakeTest.cmake | 4 + 19 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 Tests/Module/ExternalData/DataScript.dat.md5 create mode 100644 Tests/Module/ExternalData/MyScript1.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom1-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom1-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom1.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom2-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom2-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom2.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom3-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom3-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom3.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom4-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom4-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom4.cmake diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 4f84d53d1..2d5088a3a 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -99,6 +99,12 @@ calling any of the functions provided by this module. default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of content links under ``ExternalData_SOURCE_ROOT``. +.. variable:: ExternalData_CUSTOM_SCRIPT_ + + Specify a full path to a ``.cmake`` custom fetch script identified by + ```` in entries of the ``ExternalData_URL_TEMPLATES`` list. + See `Custom Fetch Scripts`_. + .. variable:: ExternalData_LINK_CONTENT The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a @@ -246,6 +252,44 @@ The following hash algorithms are supported:: Note that the hashes are used only for unique data identification and download verification. + +Custom Fetch Scripts +^^^^^^^^^^^^^^^^^^^^ + +When a data file must be fetched from one of the URL templates +specified in the ``ExternalData_URL_TEMPLATES`` variable, it is +normally downloaded using the :command:`file(DOWNLOAD)` command. +One may specify usage of a custom fetch script by using a URL +template of the form ``ExternalDataCustomScript:///``. +The ```` must be a C identifier, and the ```` must +contain the ``%(algo)`` and ``%(hash)`` placeholders. +A variable corresponding to the key, ``ExternalData_CUSTOM_SCRIPT_``, +must be set to the full path to a ``.cmake`` script file. The script +will be included to perform the actual fetch, and provided with +the following variables: + +.. variable:: ExternalData_CUSTOM_LOCATION + + When a custom fetch script is loaded, this variable is set to the + location part of the URL, which will contain the substituted hash + algorithm name and content hash value. + +.. variable:: ExternalData_CUSTOM_FILE + + When a custom fetch script is loaded, this variable is set to the + full path to a file in which the script must store the fetched + content. The name of the file is unspecified and should not be + interpreted in any way. + +The custom fetch script is expected to store fetched content in the +file or set a variable: + +.. variable:: ExternalData_CUSTOM_ERROR + + When a custom fetch script fails to fetch the requested content, + it must set this variable to a short one-line message describing + the reason for failure. + #]=======================================================================] #============================================================================= @@ -275,6 +319,37 @@ function(ExternalData_add_target target) if(NOT ExternalData_OBJECT_STORES) set(ExternalData_OBJECT_STORES ${CMAKE_BINARY_DIR}/ExternalData/Objects) endif() + set(_ExternalData_CONFIG_CODE "") + + # Store custom script configuration. + foreach(url_template IN LISTS ExternalData_URL_TEMPLATES) + if("${url_template}" MATCHES "^ExternalDataCustomScript://([^/]*)/(.*)$") + set(key "${CMAKE_MATCH_1}") + if(key MATCHES "^[A-Za-z_][A-Za-z0-9_]*$") + if(ExternalData_CUSTOM_SCRIPT_${key}) + if(IS_ABSOLUTE "${ExternalData_CUSTOM_SCRIPT_${key}}") + string(CONCAT _ExternalData_CONFIG_CODE "${_ExternalData_CONFIG_CODE}\n" + "set(ExternalData_CUSTOM_SCRIPT_${key} \"${ExternalData_CUSTOM_SCRIPT_${key}}\")") + else() + message(FATAL_ERROR + "No ExternalData_CUSTOM_SCRIPT_${key} is not set to a full path:\n" + " ${ExternalData_CUSTOM_SCRIPT_${key}}") + endif() + else() + message(FATAL_ERROR + "No ExternalData_CUSTOM_SCRIPT_${key} is set for URL template:\n" + " ${url_template}") + endif() + else() + message(FATAL_ERROR + "Bad ExternalDataCustomScript key '${key}' in URL template:\n" + " ${url_template}\n" + "The key must be a valid C identifier.") + endif() + endif() + endforeach() + + # Store configuration for use by build-time script. set(config ${CMAKE_CURRENT_BINARY_DIR}/${target}_config.cmake) configure_file(${_ExternalData_SELF_DIR}/ExternalData_config.cmake.in ${config} @ONLY) @@ -781,6 +856,30 @@ function(_ExternalData_download_file url file err_var msg_var) set("${msg_var}" "${msg}" PARENT_SCOPE) endfunction() +function(_ExternalData_custom_fetch key loc file err_var msg_var) + if(NOT ExternalData_CUSTOM_SCRIPT_${key}) + set(err 1) + set(msg "No ExternalData_CUSTOM_SCRIPT_${key} set!") + elseif(NOT EXISTS "${ExternalData_CUSTOM_SCRIPT_${key}}") + set(err 1) + set(msg "No '${ExternalData_CUSTOM_SCRIPT_${key}}' exists!") + else() + set(ExternalData_CUSTOM_LOCATION "${loc}") + set(ExternalData_CUSTOM_FILE "${file}") + unset(ExternalData_CUSTOM_ERROR) + include("${ExternalData_CUSTOM_SCRIPT_${key}}") + if(DEFINED ExternalData_CUSTOM_ERROR) + set(err 1) + set(msg "${ExternalData_CUSTOM_ERROR}") + else() + set(err 0) + set(msg "no error") + endif() + endif() + set("${err_var}" "${err}" PARENT_SCOPE) + set("${msg_var}" "${msg}" PARENT_SCOPE) +endfunction() + function(_ExternalData_download_object name hash algo var_obj) # Search all object stores for an existing object. foreach(dir ${ExternalData_OBJECT_STORES}) @@ -804,7 +903,11 @@ function(_ExternalData_download_object name hash algo var_obj) string(REPLACE "%(hash)" "${hash}" url_tmp "${url_template}") string(REPLACE "%(algo)" "${algo}" url "${url_tmp}") message(STATUS "Fetching \"${url}\"") - _ExternalData_download_file("${url}" "${tmp}" err errMsg) + if(url MATCHES "^ExternalDataCustomScript://([A-Za-z_][A-Za-z0-9_]*)/(.*)$") + _ExternalData_custom_fetch("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" "${tmp}" err errMsg) + else() + _ExternalData_download_file("${url}" "${tmp}" err errMsg) + endif() set(tried "${tried}\n ${url}") if(err) set(tried "${tried} (${errMsg})") diff --git a/Modules/ExternalData_config.cmake.in b/Modules/ExternalData_config.cmake.in index 0858f5368..4434e4bf6 100644 --- a/Modules/ExternalData_config.cmake.in +++ b/Modules/ExternalData_config.cmake.in @@ -2,3 +2,4 @@ set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@") set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@") set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@") set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@") +@_ExternalData_CONFIG_CODE@ diff --git a/Tests/Module/ExternalData/CMakeLists.txt b/Tests/Module/ExternalData/CMakeLists.txt index ebca48e44..f99f6afc5 100644 --- a/Tests/Module/ExternalData/CMakeLists.txt +++ b/Tests/Module/ExternalData/CMakeLists.txt @@ -10,7 +10,9 @@ if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "^/") endif() set(ExternalData_URL_TEMPLATES "file://${slash}${CMAKE_CURRENT_SOURCE_DIR}/%(algo)/%(hash)" + "ExternalDataCustomScript://MyScript1/%(algo)/%(hash)" ) +set(ExternalData_CUSTOM_SCRIPT_MyScript1 "${CMAKE_CURRENT_SOURCE_DIR}/MyScript1.cmake") set(ExternalData_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ExternalData") file(REMOVE_RECURSE ${ExternalData_BINARY_ROOT}) # clean test @@ -23,6 +25,7 @@ ExternalData_Add_Test(Data1 COMMAND ${CMAKE_COMMAND} -D Data=DATA{Data.dat} ${Data1CheckSpaces} + -D DataScript=DATA{DataScript.dat} -D DataMissing=DATA{DataMissing.dat} -D DataMissingWithAssociated=DATA{DataMissing.dat,Data.dat} -D SeriesA=DATA{SeriesA.dat,:} diff --git a/Tests/Module/ExternalData/Data1Check.cmake b/Tests/Module/ExternalData/Data1Check.cmake index 485b5c6c2..a7aa4aee6 100644 --- a/Tests/Module/ExternalData/Data1Check.cmake +++ b/Tests/Module/ExternalData/Data1Check.cmake @@ -8,6 +8,10 @@ if(DEFINED DataSpace) message(SEND_ERROR "Input file:\n ${DataSpace}\ndoes not have expected content, but [[${lines}]]") endif() endif() +file(STRINGS "${DataScript}" lines LIMIT_INPUT 1024) +if(NOT "x${lines}" STREQUAL "xDataScript") + message(SEND_ERROR "Input file:\n ${DataScript}\ndoes not have expected content, but [[${lines}]]") +endif() if(DataMissing) if(EXISTS "${DataMissing}") message(SEND_ERROR diff --git a/Tests/Module/ExternalData/DataScript.dat.md5 b/Tests/Module/ExternalData/DataScript.dat.md5 new file mode 100644 index 000000000..74b461607 --- /dev/null +++ b/Tests/Module/ExternalData/DataScript.dat.md5 @@ -0,0 +1 @@ +fd95c03719e8626c0d10a818f9996dc5 diff --git a/Tests/Module/ExternalData/MyScript1.cmake b/Tests/Module/ExternalData/MyScript1.cmake new file mode 100644 index 000000000..242c64d9f --- /dev/null +++ b/Tests/Module/ExternalData/MyScript1.cmake @@ -0,0 +1,5 @@ +if(ExternalData_CUSTOM_LOCATION STREQUAL "MD5/fd95c03719e8626c0d10a818f9996dc5") + file(WRITE "${ExternalData_CUSTOM_FILE}" "DataScript") +else() + set(ExternalData_CUSTOM_ERROR "no ${ExternalData_CUSTOM_LOCATION} known") +endif() diff --git a/Tests/RunCMake/ExternalData/BadCustom1-result.txt b/Tests/RunCMake/ExternalData/BadCustom1-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt new file mode 100644 index 000000000..5d2986da5 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt @@ -0,0 +1,9 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + Bad ExternalDataCustomScript key '0BadKey' in URL template: + + ExternalDataCustomScript://0BadKey/%\(algo\)/%\(hash\) + + The key must be a valid C identifier. +Call Stack \(most recent call first\): + BadCustom1.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom1.cmake b/Tests/RunCMake/ExternalData/BadCustom1.cmake new file mode 100644 index 000000000..ec94fc157 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://0BadKey/%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom2-result.txt b/Tests/RunCMake/ExternalData/BadCustom2-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt new file mode 100644 index 000000000..4d59ca9fa --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt @@ -0,0 +1,9 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + Bad ExternalDataCustomScript key '' in URL template: + + ExternalDataCustomScript:///%\(algo\)/%\(hash\) + + The key must be a valid C identifier. +Call Stack \(most recent call first\): + BadCustom2.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom2.cmake b/Tests/RunCMake/ExternalData/BadCustom2.cmake new file mode 100644 index 000000000..1ed764610 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript:///%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom3-result.txt b/Tests/RunCMake/ExternalData/BadCustom3-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt new file mode 100644 index 000000000..460814b9c --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt @@ -0,0 +1,7 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + No ExternalData_CUSTOM_SCRIPT_MissingKey is set for URL template: + + ExternalDataCustomScript://MissingKey/%\(algo\)/%\(hash\) +Call Stack \(most recent call first\): + BadCustom3.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom3.cmake b/Tests/RunCMake/ExternalData/BadCustom3.cmake new file mode 100644 index 000000000..b4f2fb8ba --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://MissingKey/%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom4-result.txt b/Tests/RunCMake/ExternalData/BadCustom4-result.txt new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt new file mode 100644 index 000000000..b83bceefa --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt @@ -0,0 +1,7 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + No ExternalData_CUSTOM_SCRIPT_RelPathKey is not set to a full path: + + RelPathScript.cmake +Call Stack \(most recent call first\): + BadCustom4.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom4.cmake b/Tests/RunCMake/ExternalData/BadCustom4.cmake new file mode 100644 index 000000000..0cc552172 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4.cmake @@ -0,0 +1,6 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://RelPathKey/%(algo)/%(hash)" + ) +set(ExternalData_CUSTOM_SCRIPT_RelPathKey "RelPathScript.cmake") +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake index 04e3d593f..7afd2893b 100644 --- a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake @@ -1,5 +1,9 @@ include(RunCMake) +run_cmake(BadCustom1) +run_cmake(BadCustom2) +run_cmake(BadCustom3) +run_cmake(BadCustom4) run_cmake(BadHashAlgo1) run_cmake(BadOption1) run_cmake(BadOption2) From 829fc5ad791d6f292d87db5e98c8d5ed2ca9123b Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 15 Jan 2015 11:03:33 -0500 Subject: [PATCH 7/7] Help: Add notes for topic 'ExternalData-custom-download' --- Help/release/dev/ExternalData-custom-download.rst | 7 +++++++ Modules/ExternalData.cmake | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 Help/release/dev/ExternalData-custom-download.rst diff --git a/Help/release/dev/ExternalData-custom-download.rst b/Help/release/dev/ExternalData-custom-download.rst new file mode 100644 index 000000000..c40f4f7f3 --- /dev/null +++ b/Help/release/dev/ExternalData-custom-download.rst @@ -0,0 +1,7 @@ +ExternalData-custom-download +---------------------------- + +* The :module:`ExternalData` module learned to support + :ref:`Custom Fetch Scripts `. + This allows projects to specify custom ``.cmake`` scripts for + fetching data objects during the build. diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 2d5088a3a..741db81b8 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -253,6 +253,8 @@ The following hash algorithms are supported:: Note that the hashes are used only for unique data identification and download verification. +.. _`ExternalData Custom Fetch Scripts`: + Custom Fetch Scripts ^^^^^^^^^^^^^^^^^^^^