|
|
|
|
@ -1,189 +1,301 @@
|
|
|
|
|
#.rst:
|
|
|
|
|
# ExternalData
|
|
|
|
|
# ------------
|
|
|
|
|
#
|
|
|
|
|
# Manage data files stored outside source tree
|
|
|
|
|
#
|
|
|
|
|
# Use this module to unambiguously reference data files stored outside
|
|
|
|
|
# the source tree and fetch them at build time from arbitrary local and
|
|
|
|
|
# remote content-addressed locations. Functions provided by this module
|
|
|
|
|
# recognize arguments with the syntax ``DATA{<name>}`` as references to
|
|
|
|
|
# external data, replace them with full paths to local copies of those
|
|
|
|
|
# data, and create build rules to fetch and update the local copies.
|
|
|
|
|
#
|
|
|
|
|
# The ``DATA{}`` syntax is literal and the ``<name>`` is a full or relative path
|
|
|
|
|
# within the source tree. The source tree must contain either a real
|
|
|
|
|
# data file at ``<name>`` or a "content link" at ``<name><ext>`` containing a
|
|
|
|
|
# hash of the real file using a hash algorithm corresponding to ``<ext>``.
|
|
|
|
|
# For example, the argument ``DATA{img.png}`` may be satisfied by either a
|
|
|
|
|
# real ``img.png`` file in the current source directory or a ``img.png.md5``
|
|
|
|
|
# file containing its MD5 sum.
|
|
|
|
|
#
|
|
|
|
|
# The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}``
|
|
|
|
|
# references in its arguments and constructs a new list of arguments:
|
|
|
|
|
#
|
|
|
|
|
# .. code-block:: cmake
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_Expand_Arguments(
|
|
|
|
|
# <target> # Name of data management target
|
|
|
|
|
# <outVar> # Output variable
|
|
|
|
|
# [args...] # Input arguments, DATA{} allowed
|
|
|
|
|
# )
|
|
|
|
|
#
|
|
|
|
|
# It replaces each ``DATA{}`` reference in an argument with the full path of
|
|
|
|
|
# a real data file on disk that will exist after the ``<target>`` builds.
|
|
|
|
|
#
|
|
|
|
|
# The ``ExternalData_Add_Test`` function wraps around the CMake
|
|
|
|
|
# :command:`add_test` command but supports ``DATA{}`` references in
|
|
|
|
|
# its arguments:
|
|
|
|
|
#
|
|
|
|
|
# .. code-block:: cmake
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_Add_Test(
|
|
|
|
|
# <target> # Name of data management target
|
|
|
|
|
# ... # Arguments of add_test(), DATA{} allowed
|
|
|
|
|
# )
|
|
|
|
|
#
|
|
|
|
|
# It passes its arguments through ``ExternalData_Expand_Arguments`` and then
|
|
|
|
|
# invokes the :command:`add_test` command using the results.
|
|
|
|
|
#
|
|
|
|
|
# The ``ExternalData_Add_Target`` function creates a custom target to
|
|
|
|
|
# manage local instances of data files stored externally:
|
|
|
|
|
#
|
|
|
|
|
# .. code-block:: cmake
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_Add_Target(
|
|
|
|
|
# <target> # Name of data management target
|
|
|
|
|
# )
|
|
|
|
|
#
|
|
|
|
|
# It creates custom commands in the target as necessary to make data
|
|
|
|
|
# files available for each ``DATA{}`` reference previously evaluated by
|
|
|
|
|
# other functions provided by this module. A list of URL templates may
|
|
|
|
|
# be provided in the variable ``ExternalData_URL_TEMPLATES`` using the
|
|
|
|
|
# placeholders ``%(algo)`` and ``%(hash)`` in each template. Data fetch
|
|
|
|
|
# rules try each URL template in order by substituting the hash
|
|
|
|
|
# algorithm name for ``%(algo)`` and the hash value for ``%(hash)``.
|
|
|
|
|
#
|
|
|
|
|
# The following hash algorithms are supported::
|
|
|
|
|
#
|
|
|
|
|
# %(algo) <ext> Description
|
|
|
|
|
# ------- ----- -----------
|
|
|
|
|
# MD5 .md5 Message-Digest Algorithm 5, RFC 1321
|
|
|
|
|
# SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174
|
|
|
|
|
# SHA224 .sha224 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
# SHA256 .sha256 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
# SHA384 .sha384 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
# SHA512 .sha512 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
#
|
|
|
|
|
# Note that the hashes are used only for unique data identification and
|
|
|
|
|
# download verification.
|
|
|
|
|
#
|
|
|
|
|
# Example usage:
|
|
|
|
|
#
|
|
|
|
|
# .. code-block:: cmake
|
|
|
|
|
#
|
|
|
|
|
# include(ExternalData)
|
|
|
|
|
# set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)"
|
|
|
|
|
# "file:////host/share/%(algo)/%(hash)"
|
|
|
|
|
# "http://data.org/%(algo)/%(hash)")
|
|
|
|
|
# ExternalData_Add_Test(MyData
|
|
|
|
|
# NAME MyTest
|
|
|
|
|
# COMMAND MyExe DATA{MyInput.png}
|
|
|
|
|
# )
|
|
|
|
|
# ExternalData_Add_Target(MyData)
|
|
|
|
|
#
|
|
|
|
|
# When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be
|
|
|
|
|
# replaced by the full path to a real instance of the data file
|
|
|
|
|
# ``MyInput.png`` on disk. If the source tree contains a content link
|
|
|
|
|
# such as ``MyInput.png.md5`` then the ``MyData`` target creates a real
|
|
|
|
|
# ``MyInput.png`` in the build tree.
|
|
|
|
|
#
|
|
|
|
|
# The ``DATA{}`` syntax can be told to fetch a file series using the form
|
|
|
|
|
# ``DATA{<name>,:}``, where the ``:`` is literal. If the source tree
|
|
|
|
|
# contains a group of files or content links named like a series then a
|
|
|
|
|
# reference to one member adds rules to fetch all of them. Although all
|
|
|
|
|
# members of a series are fetched, only the file originally named by the
|
|
|
|
|
# ``DATA{}`` argument is substituted for it. The default configuration
|
|
|
|
|
# recognizes file series names ending with ``#.ext``, ``_#.ext``, ``.#.ext``,
|
|
|
|
|
# or ``-#.ext`` where ``#`` is a sequence of decimal digits and ``.ext`` is
|
|
|
|
|
# any single extension. Configure it with a regex that parses ``<number>``
|
|
|
|
|
# and ``<suffix>`` parts from the end of ``<name>``::
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_SERIES_PARSE = regex of the form (<number>)(<suffix>)$
|
|
|
|
|
#
|
|
|
|
|
# For more complicated cases set::
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_SERIES_PARSE = regex with at least two () groups
|
|
|
|
|
# ExternalData_SERIES_PARSE_PREFIX = <prefix> regex group number, if any
|
|
|
|
|
# ExternalData_SERIES_PARSE_NUMBER = <number> regex group number
|
|
|
|
|
# ExternalData_SERIES_PARSE_SUFFIX = <suffix> regex group number
|
|
|
|
|
#
|
|
|
|
|
# Configure series number matching with a regex that matches the
|
|
|
|
|
# ``<number>`` part of series members named ``<prefix><number><suffix>``::
|
|
|
|
|
#
|
|
|
|
|
# ExternalData_SERIES_MATCH = regex matching <number> in all series members
|
|
|
|
|
#
|
|
|
|
|
# Note that the ``<suffix>`` of a series does not include a hash-algorithm
|
|
|
|
|
# extension.
|
|
|
|
|
#
|
|
|
|
|
# The ``DATA{}`` syntax can alternatively match files associated with the
|
|
|
|
|
# named file and contained in the same directory. Associated files may
|
|
|
|
|
# be specified by options using the syntax
|
|
|
|
|
# ``DATA{<name>,<opt1>,<opt2>,...}``. Each option may specify one file by
|
|
|
|
|
# name or specify a regular expression to match file names using the
|
|
|
|
|
# syntax ``REGEX:<regex>``. For example, the arguments::
|
|
|
|
|
#
|
|
|
|
|
# DATA{MyData/MyInput.mhd,MyInput.img} # File pair
|
|
|
|
|
# DATA{MyData/MyFrames00.png,REGEX:MyFrames[0-9]+\\.png} # Series
|
|
|
|
|
#
|
|
|
|
|
# will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but
|
|
|
|
|
# ensure that the associated files are present next to them.
|
|
|
|
|
#
|
|
|
|
|
# The ``DATA{}`` syntax may reference a directory using a trailing slash and
|
|
|
|
|
# a list of associated files. The form ``DATA{<name>/,<opt1>,<opt2>,...}``
|
|
|
|
|
# adds rules to fetch any files in the directory that match one of the
|
|
|
|
|
# associated file options. For example, the argument
|
|
|
|
|
# ``DATA{MyDataDir/,REGEX:.*}`` will pass the full path to a ``MyDataDir``
|
|
|
|
|
# directory on the command line and ensure that the directory contains
|
|
|
|
|
# files corresponding to every file or content link in the ``MyDataDir``
|
|
|
|
|
# source directory.
|
|
|
|
|
#
|
|
|
|
|
# The variable ``ExternalData_LINK_CONTENT`` may be set to the name of a
|
|
|
|
|
# supported hash algorithm to enable automatic conversion of real data
|
|
|
|
|
# files referenced by the ``DATA{}`` syntax into content links. For each
|
|
|
|
|
# such ``<file>`` a content link named ``<file><ext>`` is created. The
|
|
|
|
|
# original file is renamed to the form ``.ExternalData_<algo>_<hash>`` to
|
|
|
|
|
# stage it for future transmission to one of the locations in the list
|
|
|
|
|
# of URL templates (by means outside the scope of this module). The
|
|
|
|
|
# data fetch rule created for the content link will use the staged
|
|
|
|
|
# object if it cannot be found using any URL template.
|
|
|
|
|
#
|
|
|
|
|
# The variable ``ExternalData_OBJECT_STORES`` may be set to a list of local
|
|
|
|
|
# directories that store objects using the layout ``<dir>/%(algo)/%(hash)``.
|
|
|
|
|
# These directories will be searched first for a needed object. If the
|
|
|
|
|
# object is not available in any store then it will be fetched remotely
|
|
|
|
|
# using the URL templates and added to the first local store listed. If
|
|
|
|
|
# no stores are specified the default is a location inside the build
|
|
|
|
|
# tree.
|
|
|
|
|
#
|
|
|
|
|
# The variable ``ExternalData_SOURCE_ROOT`` may be set to the highest source
|
|
|
|
|
# directory containing any path named by a ``DATA{}`` reference. The
|
|
|
|
|
# default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and
|
|
|
|
|
# ``CMAKE_SOURCE_DIR`` must refer to directories within a single source
|
|
|
|
|
# distribution (e.g. they come together in one tarball).
|
|
|
|
|
#
|
|
|
|
|
# The variable ``ExternalData_BINARY_ROOT`` may be set to the directory to
|
|
|
|
|
# hold the real data files named by expanded ``DATA{}`` references. The
|
|
|
|
|
# default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of
|
|
|
|
|
# content links under ``ExternalData_SOURCE_ROOT``.
|
|
|
|
|
#
|
|
|
|
|
# Variables ``ExternalData_TIMEOUT_INACTIVITY`` and
|
|
|
|
|
# ``ExternalData_TIMEOUT_ABSOLUTE`` set the download inactivity and absolute
|
|
|
|
|
# timeouts, in seconds. The defaults are 60 seconds and 300 seconds,
|
|
|
|
|
# respectively. Set either timeout to 0 seconds to disable enforcement.
|
|
|
|
|
#[=======================================================================[.rst:
|
|
|
|
|
ExternalData
|
|
|
|
|
------------
|
|
|
|
|
|
|
|
|
|
.. only:: html
|
|
|
|
|
|
|
|
|
|
.. contents::
|
|
|
|
|
|
|
|
|
|
Manage data files stored outside source tree
|
|
|
|
|
|
|
|
|
|
Introduction
|
|
|
|
|
^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
Use this module to unambiguously reference data files stored outside
|
|
|
|
|
the source tree and fetch them at build time from arbitrary local and
|
|
|
|
|
remote content-addressed locations. Functions provided by this module
|
|
|
|
|
recognize arguments with the syntax ``DATA{<name>}`` as references to
|
|
|
|
|
external data, replace them with full paths to local copies of those
|
|
|
|
|
data, and create build rules to fetch and update the local copies.
|
|
|
|
|
|
|
|
|
|
For example:
|
|
|
|
|
|
|
|
|
|
.. code-block:: cmake
|
|
|
|
|
|
|
|
|
|
include(ExternalData)
|
|
|
|
|
set(ExternalData_URL_TEMPLATES "file:///local/%(algo)/%(hash)"
|
|
|
|
|
"file:////host/share/%(algo)/%(hash)"
|
|
|
|
|
"http://data.org/%(algo)/%(hash)")
|
|
|
|
|
ExternalData_Add_Test(MyData
|
|
|
|
|
NAME MyTest
|
|
|
|
|
COMMAND MyExe DATA{MyInput.png}
|
|
|
|
|
)
|
|
|
|
|
ExternalData_Add_Target(MyData)
|
|
|
|
|
|
|
|
|
|
When test ``MyTest`` runs the ``DATA{MyInput.png}`` argument will be
|
|
|
|
|
replaced by the full path to a real instance of the data file
|
|
|
|
|
``MyInput.png`` on disk. If the source tree contains a content link
|
|
|
|
|
such as ``MyInput.png.md5`` then the ``MyData`` target creates a real
|
|
|
|
|
``MyInput.png`` in the build tree.
|
|
|
|
|
|
|
|
|
|
Module Functions
|
|
|
|
|
^^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
.. command:: ExternalData_Expand_Arguments
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_Expand_Arguments`` function evaluates ``DATA{}``
|
|
|
|
|
references in its arguments and constructs a new list of arguments::
|
|
|
|
|
|
|
|
|
|
ExternalData_Expand_Arguments(
|
|
|
|
|
<target> # Name of data management target
|
|
|
|
|
<outVar> # Output variable
|
|
|
|
|
[args...] # Input arguments, DATA{} allowed
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
It replaces each ``DATA{}`` reference in an argument with the full path of
|
|
|
|
|
a real data file on disk that will exist after the ``<target>`` builds.
|
|
|
|
|
|
|
|
|
|
.. command:: ExternalData_Add_Test
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_Add_Test`` function wraps around the CMake
|
|
|
|
|
:command:`add_test` command but supports ``DATA{}`` references in
|
|
|
|
|
its arguments::
|
|
|
|
|
|
|
|
|
|
ExternalData_Add_Test(
|
|
|
|
|
<target> # Name of data management target
|
|
|
|
|
... # Arguments of add_test(), DATA{} allowed
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
It passes its arguments through ``ExternalData_Expand_Arguments`` and then
|
|
|
|
|
invokes the :command:`add_test` command using the results.
|
|
|
|
|
|
|
|
|
|
.. command:: ExternalData_Add_Target
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_Add_Target`` function creates a custom target to
|
|
|
|
|
manage local instances of data files stored externally::
|
|
|
|
|
|
|
|
|
|
ExternalData_Add_Target(
|
|
|
|
|
<target> # Name of data management target
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
It creates custom commands in the target as necessary to make data
|
|
|
|
|
files available for each ``DATA{}`` reference previously evaluated by
|
|
|
|
|
other functions provided by this module.
|
|
|
|
|
Data files may be fetched from one of the URL templates specified in
|
|
|
|
|
the ``ExternalData_URL_TEMPLATES`` variable, or may be found locally
|
|
|
|
|
in one of the paths specified in the ``ExternalData_OBJECT_STORES``
|
|
|
|
|
variable.
|
|
|
|
|
|
|
|
|
|
Module Variables
|
|
|
|
|
^^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
The following variables configure behavior. They should be set before
|
|
|
|
|
calling any of the functions provided by this module.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_BINARY_ROOT
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_BINARY_ROOT`` variable may be set to the directory to
|
|
|
|
|
hold the real data files named by expanded ``DATA{}`` references. The
|
|
|
|
|
default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of
|
|
|
|
|
content links under ``ExternalData_SOURCE_ROOT``.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_CUSTOM_SCRIPT_<key>
|
|
|
|
|
|
|
|
|
|
Specify a full path to a ``.cmake`` custom fetch script identified by
|
|
|
|
|
``<key>`` in entries of the ``ExternalData_URL_TEMPLATES`` list.
|
|
|
|
|
See `Custom Fetch Scripts`_.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_LINK_CONTENT
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a
|
|
|
|
|
supported hash algorithm to enable automatic conversion of real data
|
|
|
|
|
files referenced by the ``DATA{}`` syntax into content links. For each
|
|
|
|
|
such ``<file>`` a content link named ``<file><ext>`` is created. The
|
|
|
|
|
original file is renamed to the form ``.ExternalData_<algo>_<hash>`` to
|
|
|
|
|
stage it for future transmission to one of the locations in the list
|
|
|
|
|
of URL templates (by means outside the scope of this module). The
|
|
|
|
|
data fetch rule created for the content link will use the staged
|
|
|
|
|
object if it cannot be found using any URL template.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_OBJECT_STORES
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local
|
|
|
|
|
directories that store objects using the layout ``<dir>/%(algo)/%(hash)``.
|
|
|
|
|
These directories will be searched first for a needed object. If the
|
|
|
|
|
object is not available in any store then it will be fetched remotely
|
|
|
|
|
using the URL templates and added to the first local store listed. If
|
|
|
|
|
no stores are specified the default is a location inside the build
|
|
|
|
|
tree.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_SERIES_PARSE
|
|
|
|
|
ExternalData_SERIES_PARSE_PREFIX
|
|
|
|
|
ExternalData_SERIES_PARSE_NUMBER
|
|
|
|
|
ExternalData_SERIES_PARSE_SUFFIX
|
|
|
|
|
ExternalData_SERIES_MATCH
|
|
|
|
|
|
|
|
|
|
See `Referencing File Series`_.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_SOURCE_ROOT
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_SOURCE_ROOT`` variable may be set to the highest source
|
|
|
|
|
directory containing any path named by a ``DATA{}`` reference. The
|
|
|
|
|
default is ``CMAKE_SOURCE_DIR``. ``ExternalData_SOURCE_ROOT`` and
|
|
|
|
|
``CMAKE_SOURCE_DIR`` must refer to directories within a single source
|
|
|
|
|
distribution (e.g. they come together in one tarball).
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_TIMEOUT_ABSOLUTE
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_TIMEOUT_ABSOLUTE`` variable sets the download
|
|
|
|
|
absolute timeout, in seconds, with a default of ``300`` seconds.
|
|
|
|
|
Set to ``0`` to disable enforcement.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_TIMEOUT_INACTIVITY
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_TIMEOUT_INACTIVITY`` variable sets the download
|
|
|
|
|
inactivity timeout, in seconds, with a default of ``60`` seconds.
|
|
|
|
|
Set to ``0`` to disable enforcement.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_URL_TEMPLATES
|
|
|
|
|
|
|
|
|
|
The ``ExternalData_URL_TEMPLATES`` may be set to provide a list of
|
|
|
|
|
of URL templates using the placeholders ``%(algo)`` and ``%(hash)``
|
|
|
|
|
in each template. Data fetch rules try each URL template in order
|
|
|
|
|
by substituting the hash algorithm name for ``%(algo)`` and the hash
|
|
|
|
|
value for ``%(hash)``.
|
|
|
|
|
|
|
|
|
|
Referencing Files
|
|
|
|
|
^^^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
Referencing Single Files
|
|
|
|
|
""""""""""""""""""""""""
|
|
|
|
|
|
|
|
|
|
The ``DATA{}`` syntax is literal and the ``<name>`` is a full or relative path
|
|
|
|
|
within the source tree. The source tree must contain either a real
|
|
|
|
|
data file at ``<name>`` or a "content link" at ``<name><ext>`` containing a
|
|
|
|
|
hash of the real file using a hash algorithm corresponding to ``<ext>``.
|
|
|
|
|
For example, the argument ``DATA{img.png}`` may be satisfied by either a
|
|
|
|
|
real ``img.png`` file in the current source directory or a ``img.png.md5``
|
|
|
|
|
file containing its MD5 sum.
|
|
|
|
|
|
|
|
|
|
Referencing File Series
|
|
|
|
|
"""""""""""""""""""""""
|
|
|
|
|
|
|
|
|
|
The ``DATA{}`` syntax can be told to fetch a file series using the form
|
|
|
|
|
``DATA{<name>,:}``, where the ``:`` is literal. If the source tree
|
|
|
|
|
contains a group of files or content links named like a series then a
|
|
|
|
|
reference to one member adds rules to fetch all of them. Although all
|
|
|
|
|
members of a series are fetched, only the file originally named by the
|
|
|
|
|
``DATA{}`` argument is substituted for it. The default configuration
|
|
|
|
|
recognizes file series names ending with ``#.ext``, ``_#.ext``, ``.#.ext``,
|
|
|
|
|
or ``-#.ext`` where ``#`` is a sequence of decimal digits and ``.ext`` is
|
|
|
|
|
any single extension. Configure it with a regex that parses ``<number>``
|
|
|
|
|
and ``<suffix>`` parts from the end of ``<name>``::
|
|
|
|
|
|
|
|
|
|
ExternalData_SERIES_PARSE = regex of the form (<number>)(<suffix>)$
|
|
|
|
|
|
|
|
|
|
For more complicated cases set::
|
|
|
|
|
|
|
|
|
|
ExternalData_SERIES_PARSE = regex with at least two () groups
|
|
|
|
|
ExternalData_SERIES_PARSE_PREFIX = <prefix> regex group number, if any
|
|
|
|
|
ExternalData_SERIES_PARSE_NUMBER = <number> regex group number
|
|
|
|
|
ExternalData_SERIES_PARSE_SUFFIX = <suffix> regex group number
|
|
|
|
|
|
|
|
|
|
Configure series number matching with a regex that matches the
|
|
|
|
|
``<number>`` part of series members named ``<prefix><number><suffix>``::
|
|
|
|
|
|
|
|
|
|
ExternalData_SERIES_MATCH = regex matching <number> in all series members
|
|
|
|
|
|
|
|
|
|
Note that the ``<suffix>`` of a series does not include a hash-algorithm
|
|
|
|
|
extension.
|
|
|
|
|
|
|
|
|
|
Referencing Associated Files
|
|
|
|
|
""""""""""""""""""""""""""""
|
|
|
|
|
|
|
|
|
|
The ``DATA{}`` syntax can alternatively match files associated with the
|
|
|
|
|
named file and contained in the same directory. Associated files may
|
|
|
|
|
be specified by options using the syntax
|
|
|
|
|
``DATA{<name>,<opt1>,<opt2>,...}``. Each option may specify one file by
|
|
|
|
|
name or specify a regular expression to match file names using the
|
|
|
|
|
syntax ``REGEX:<regex>``. For example, the arguments::
|
|
|
|
|
|
|
|
|
|
DATA{MyData/MyInput.mhd,MyInput.img} # File pair
|
|
|
|
|
DATA{MyData/MyFrames00.png,REGEX:MyFrames[0-9]+\\.png} # Series
|
|
|
|
|
|
|
|
|
|
will pass ``MyInput.mha`` and ``MyFrames00.png`` on the command line but
|
|
|
|
|
ensure that the associated files are present next to them.
|
|
|
|
|
|
|
|
|
|
Referencing Directories
|
|
|
|
|
"""""""""""""""""""""""
|
|
|
|
|
|
|
|
|
|
The ``DATA{}`` syntax may reference a directory using a trailing slash and
|
|
|
|
|
a list of associated files. The form ``DATA{<name>/,<opt1>,<opt2>,...}``
|
|
|
|
|
adds rules to fetch any files in the directory that match one of the
|
|
|
|
|
associated file options. For example, the argument
|
|
|
|
|
``DATA{MyDataDir/,REGEX:.*}`` will pass the full path to a ``MyDataDir``
|
|
|
|
|
directory on the command line and ensure that the directory contains
|
|
|
|
|
files corresponding to every file or content link in the ``MyDataDir``
|
|
|
|
|
source directory.
|
|
|
|
|
|
|
|
|
|
Hash Algorithms
|
|
|
|
|
^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
The following hash algorithms are supported::
|
|
|
|
|
|
|
|
|
|
%(algo) <ext> Description
|
|
|
|
|
------- ----- -----------
|
|
|
|
|
MD5 .md5 Message-Digest Algorithm 5, RFC 1321
|
|
|
|
|
SHA1 .sha1 US Secure Hash Algorithm 1, RFC 3174
|
|
|
|
|
SHA224 .sha224 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
SHA256 .sha256 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
SHA384 .sha384 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
SHA512 .sha512 US Secure Hash Algorithms, RFC 4634
|
|
|
|
|
|
|
|
|
|
Note that the hashes are used only for unique data identification and
|
|
|
|
|
download verification.
|
|
|
|
|
|
|
|
|
|
.. _`ExternalData Custom Fetch Scripts`:
|
|
|
|
|
|
|
|
|
|
Custom Fetch Scripts
|
|
|
|
|
^^^^^^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
When a data file must be fetched from one of the URL templates
|
|
|
|
|
specified in the ``ExternalData_URL_TEMPLATES`` variable, it is
|
|
|
|
|
normally downloaded using the :command:`file(DOWNLOAD)` command.
|
|
|
|
|
One may specify usage of a custom fetch script by using a URL
|
|
|
|
|
template of the form ``ExternalDataCustomScript://<key>/<loc>``.
|
|
|
|
|
The ``<key>`` must be a C identifier, and the ``<loc>`` must
|
|
|
|
|
contain the ``%(algo)`` and ``%(hash)`` placeholders.
|
|
|
|
|
A variable corresponding to the key, ``ExternalData_CUSTOM_SCRIPT_<key>``,
|
|
|
|
|
must be set to the full path to a ``.cmake`` script file. The script
|
|
|
|
|
will be included to perform the actual fetch, and provided with
|
|
|
|
|
the following variables:
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_CUSTOM_LOCATION
|
|
|
|
|
|
|
|
|
|
When a custom fetch script is loaded, this variable is set to the
|
|
|
|
|
location part of the URL, which will contain the substituted hash
|
|
|
|
|
algorithm name and content hash value.
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_CUSTOM_FILE
|
|
|
|
|
|
|
|
|
|
When a custom fetch script is loaded, this variable is set to the
|
|
|
|
|
full path to a file in which the script must store the fetched
|
|
|
|
|
content. The name of the file is unspecified and should not be
|
|
|
|
|
interpreted in any way.
|
|
|
|
|
|
|
|
|
|
The custom fetch script is expected to store fetched content in the
|
|
|
|
|
file or set a variable:
|
|
|
|
|
|
|
|
|
|
.. variable:: ExternalData_CUSTOM_ERROR
|
|
|
|
|
|
|
|
|
|
When a custom fetch script fails to fetch the requested content,
|
|
|
|
|
it must set this variable to a short one-line message describing
|
|
|
|
|
the reason for failure.
|
|
|
|
|
|
|
|
|
|
#]=======================================================================]
|
|
|
|
|
|
|
|
|
|
#=============================================================================
|
|
|
|
|
# Copyright 2010-2013 Kitware, Inc.
|
|
|
|
|
# Copyright 2010-2015 Kitware, Inc.
|
|
|
|
|
#
|
|
|
|
|
# Distributed under the OSI-approved BSD License (the "License");
|
|
|
|
|
# see accompanying file Copyright.txt for details.
|
|
|
|
|
@ -209,6 +321,37 @@ function(ExternalData_add_target target)
|
|
|
|
|
if(NOT ExternalData_OBJECT_STORES)
|
|
|
|
|
set(ExternalData_OBJECT_STORES ${CMAKE_BINARY_DIR}/ExternalData/Objects)
|
|
|
|
|
endif()
|
|
|
|
|
set(_ExternalData_CONFIG_CODE "")
|
|
|
|
|
|
|
|
|
|
# Store custom script configuration.
|
|
|
|
|
foreach(url_template IN LISTS ExternalData_URL_TEMPLATES)
|
|
|
|
|
if("${url_template}" MATCHES "^ExternalDataCustomScript://([^/]*)/(.*)$")
|
|
|
|
|
set(key "${CMAKE_MATCH_1}")
|
|
|
|
|
if(key MATCHES "^[A-Za-z_][A-Za-z0-9_]*$")
|
|
|
|
|
if(ExternalData_CUSTOM_SCRIPT_${key})
|
|
|
|
|
if(IS_ABSOLUTE "${ExternalData_CUSTOM_SCRIPT_${key}}")
|
|
|
|
|
string(CONCAT _ExternalData_CONFIG_CODE "${_ExternalData_CONFIG_CODE}\n"
|
|
|
|
|
"set(ExternalData_CUSTOM_SCRIPT_${key} \"${ExternalData_CUSTOM_SCRIPT_${key}}\")")
|
|
|
|
|
else()
|
|
|
|
|
message(FATAL_ERROR
|
|
|
|
|
"No ExternalData_CUSTOM_SCRIPT_${key} is not set to a full path:\n"
|
|
|
|
|
" ${ExternalData_CUSTOM_SCRIPT_${key}}")
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
|
|
|
|
message(FATAL_ERROR
|
|
|
|
|
"No ExternalData_CUSTOM_SCRIPT_${key} is set for URL template:\n"
|
|
|
|
|
" ${url_template}")
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
|
|
|
|
message(FATAL_ERROR
|
|
|
|
|
"Bad ExternalDataCustomScript key '${key}' in URL template:\n"
|
|
|
|
|
" ${url_template}\n"
|
|
|
|
|
"The key must be a valid C identifier.")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
endforeach()
|
|
|
|
|
|
|
|
|
|
# Store configuration for use by build-time script.
|
|
|
|
|
set(config ${CMAKE_CURRENT_BINARY_DIR}/${target}_config.cmake)
|
|
|
|
|
configure_file(${_ExternalData_SELF_DIR}/ExternalData_config.cmake.in ${config} @ONLY)
|
|
|
|
|
|
|
|
|
|
@ -715,6 +858,30 @@ function(_ExternalData_download_file url file err_var msg_var)
|
|
|
|
|
set("${msg_var}" "${msg}" PARENT_SCOPE)
|
|
|
|
|
endfunction()
|
|
|
|
|
|
|
|
|
|
function(_ExternalData_custom_fetch key loc file err_var msg_var)
|
|
|
|
|
if(NOT ExternalData_CUSTOM_SCRIPT_${key})
|
|
|
|
|
set(err 1)
|
|
|
|
|
set(msg "No ExternalData_CUSTOM_SCRIPT_${key} set!")
|
|
|
|
|
elseif(NOT EXISTS "${ExternalData_CUSTOM_SCRIPT_${key}}")
|
|
|
|
|
set(err 1)
|
|
|
|
|
set(msg "No '${ExternalData_CUSTOM_SCRIPT_${key}}' exists!")
|
|
|
|
|
else()
|
|
|
|
|
set(ExternalData_CUSTOM_LOCATION "${loc}")
|
|
|
|
|
set(ExternalData_CUSTOM_FILE "${file}")
|
|
|
|
|
unset(ExternalData_CUSTOM_ERROR)
|
|
|
|
|
include("${ExternalData_CUSTOM_SCRIPT_${key}}")
|
|
|
|
|
if(DEFINED ExternalData_CUSTOM_ERROR)
|
|
|
|
|
set(err 1)
|
|
|
|
|
set(msg "${ExternalData_CUSTOM_ERROR}")
|
|
|
|
|
else()
|
|
|
|
|
set(err 0)
|
|
|
|
|
set(msg "no error")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
set("${err_var}" "${err}" PARENT_SCOPE)
|
|
|
|
|
set("${msg_var}" "${msg}" PARENT_SCOPE)
|
|
|
|
|
endfunction()
|
|
|
|
|
|
|
|
|
|
function(_ExternalData_download_object name hash algo var_obj)
|
|
|
|
|
# Search all object stores for an existing object.
|
|
|
|
|
foreach(dir ${ExternalData_OBJECT_STORES})
|
|
|
|
|
@ -738,7 +905,11 @@ function(_ExternalData_download_object name hash algo var_obj)
|
|
|
|
|
string(REPLACE "%(hash)" "${hash}" url_tmp "${url_template}")
|
|
|
|
|
string(REPLACE "%(algo)" "${algo}" url "${url_tmp}")
|
|
|
|
|
message(STATUS "Fetching \"${url}\"")
|
|
|
|
|
_ExternalData_download_file("${url}" "${tmp}" err errMsg)
|
|
|
|
|
if(url MATCHES "^ExternalDataCustomScript://([A-Za-z_][A-Za-z0-9_]*)/(.*)$")
|
|
|
|
|
_ExternalData_custom_fetch("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" "${tmp}" err errMsg)
|
|
|
|
|
else()
|
|
|
|
|
_ExternalData_download_file("${url}" "${tmp}" err errMsg)
|
|
|
|
|
endif()
|
|
|
|
|
set(tried "${tried}\n ${url}")
|
|
|
|
|
if(err)
|
|
|
|
|
set(tried "${tried} (${errMsg})")
|
|
|
|
|
|
Brad King
CMake Topic Stage