From 4ae133e09c763beb5a55302fdc1502abff6ff8d3 Mon Sep 17 00:00:00 2001 From: "Daniele E. Domenichelli" Date: Mon, 3 Nov 2014 16:14:35 +0100 Subject: [PATCH 1/4] ExternalProject: Add independent step targets When adding step targets using ExternalProject_Add_StepTargets, the STEP_TARGETS argument or the EP_STEP_TARGETS property, ExternalProject sets all the dependencies for the main project to that target. Due to this, the update target cannot be used without downloading and building all the dependencies. In order to be able to add step targets that do not depend on other external projects, this patch adds: * An optional "NO_DEPENDS" argument to the ExternalProject_Add_StepTargets function. If this argument is set, then no dependencies are set for the target (file dependencies will still be set). * A new argument INDEPENDENT_STEP_TARGETS to the ExternalProject_Add function and a new directory property EP_INDEPENDENT_STEP_TARGETS that behave like STEP_TARGETS and EP_STEP_TARGETS, but cause the ExternalProject_Add_StepTargets to be called with the NO_DEPENDS argument. --- Modules/ExternalProject.cmake | 71 ++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 3d1ab34ae..02383cbb0 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -166,6 +166,9 @@ Create custom targets to build projects in external trees ``STEP_TARGETS ...`` Generate custom targets for these steps + ``INDEPENDENT_STEP_TARGETS ...`` + Generate custom targets for these steps that do not depend on other + external projects even if a dependency is set The ``*_DIR`` options specify directories for the project, with default directories computed as follows. If the ``PREFIX`` option is given to @@ -260,18 +263,30 @@ not defined. Behavior of shell operators like ``&&`` is not defined. The ``ExternalProject_Add_StepTargets`` function generates custom targets for the steps listed:: - ExternalProject_Add_StepTargets( [step1 [step2 [...]]]) + ExternalProject_Add_StepTargets( [NO_DEPENDS] [step1 [step2 [...]]]) -If ``STEP_TARGETS`` is set then ``ExternalProject_Add_StepTargets`` is -automatically called at the end of matching calls to -``ExternalProject_Add_Step``. Pass ``STEP_TARGETS`` explicitly to +If ``NO_DEPENDS`` is set, the target will not depend on the +dependencies of the complete project. This is usually safe to use for +the download, update, and patch steps that do not require that all the +dependencies are updated and built. Using ``NO_DEPENDS`` for other +of the default steps might break parallel builds, so you should avoid, +it. For custom steps, you should consider whether or not the custom +commands requires that the dependencies are configured, built and +installed. + +If ``STEP_TARGETS`` or ``INDEPENDENT_STEP_TARGETS`` is set then +``ExternalProject_Add_StepTargets`` is automatically called at the end +of matching calls to ``ExternalProject_Add_Step``. Pass +``STEP_TARGETS`` or ``INDEPENDENT_STEP_TARGETS`` explicitly to individual ``ExternalProject_Add`` calls, or implicitly to all -``ExternalProject_Add`` calls by setting the directory property -``EP_STEP_TARGETS``. +``ExternalProject_Add`` calls by setting the directory properties +``EP_STEP_TARGETS`` and ``EP_INDEPENDENT_STEP_TARGETS``. The +``INDEPENDENT`` version of the argument and of the property will call +``ExternalProject_Add_StepTargets`` with the ``NO_DEPENDS`` argument. -If ``STEP_TARGETS`` is not set, clients may still manually call -``ExternalProject_Add_StepTargets`` after calling -``ExternalProject_Add`` or ``ExternalProject_Add_Step``. +If ``STEP_TARGETS`` and ``INDEPENDENT_STEP_TARGETS`` are not set, +clients may still manually call ``ExternalProject_Add_StepTargets`` +after calling ``ExternalProject_Add`` or ``ExternalProject_Add_Step``. This functionality is provided to make it easy to drive the steps independently of each other by specifying targets on build command @@ -397,10 +412,19 @@ define_property(DIRECTORY PROPERTY "EP_STEP_TARGETS" INHERITED BRIEF_DOCS "List of ExternalProject steps that automatically get corresponding targets" FULL_DOCS + "These targets will be dependent on the main target dependencies" "See documentation of the ExternalProject_Add_StepTargets() function in the " "ExternalProject module." ) +define_property(DIRECTORY PROPERTY "EP_INDEPENDENT_STEP_TARGETS" INHERITED + BRIEF_DOCS + "List of ExternalProject steps that automatically get corresponding targets" + FULL_DOCS + "These targets will not be dependent on the main target dependencies" + "See documentation of the ExternalProject_Add_StepTargets() function in the " + "ExternalProject module." + ) function(_ep_write_gitclone_script script_filename source_dir git_EXECUTABLE git_repository git_tag git_submodules src_name work_dir gitclone_infofile gitclone_stampfile) file(WRITE ${script_filename} @@ -1283,8 +1307,14 @@ endfunction() function(ExternalProject_Add_StepTargets name) set(steps ${ARGN}) - + if("${ARGV1}" STREQUAL "NO_DEPENDS") + set(no_deps 1) + list(REMOVE_AT steps 0) + endif() foreach(step ${steps}) + if(no_deps AND "${step}" MATCHES "^(configure|build|install|test)$") + message(AUTHOR_WARNING "Using NO_DEPENDS for \"${step}\" step might break parallel builds") + endif() _ep_get_step_stampfile(${name} ${step} stamp_file) add_custom_target(${name}-${step} DEPENDS ${stamp_file}) @@ -1292,10 +1322,12 @@ function(ExternalProject_Add_StepTargets name) set_property(TARGET ${name}-${step} PROPERTY FOLDER "ExternalProjectTargets/${name}") # Depend on other external projects (target-level). - get_property(deps TARGET ${name} PROPERTY _EP_DEPENDS) - foreach(arg IN LISTS deps) - add_dependencies(${name}-${step} ${arg}) - endforeach() + if(NOT no_deps) + get_property(deps TARGET ${name} PROPERTY _EP_DEPENDS) + foreach(arg IN LISTS deps) + add_dependencies(${name}-${step} ${arg}) + endforeach() + endif() endforeach() endfunction() @@ -1398,6 +1430,17 @@ function(ExternalProject_Add_Step name step) break() endif() endforeach() + + get_property(independent_step_targets TARGET ${name} PROPERTY _EP_INDEPENDENT_STEP_TARGETS) + if(NOT independent_step_targets) + get_property(independent_step_targets DIRECTORY PROPERTY EP_INDEPENDENT_STEP_TARGETS) + endif() + foreach(st ${independent_step_targets}) + if("${st}" STREQUAL "${step}") + ExternalProject_Add_StepTargets(${name} NO_DEPENDS ${step}) + break() + endif() + endforeach() endfunction() From f598f1aa836f7c9a60bc59b53ebbc2faae24f5a4 Mon Sep 17 00:00:00 2001 From: "Daniele E. Domenichelli" Date: Mon, 3 Nov 2014 16:14:49 +0100 Subject: [PATCH 2/4] ExternalProject: Add ExternalProject_Add_StepDependencies function The ExternalProject_Add_StepDependencies function add some dependencies for some external project step. The syntax is: ExternalProject_Add_Step_Dependencies( [target1 [target2 [...]]]) This function takes care to set both target and file level dependencies, and will ensure that parallel builds will not break. It should be used instead of add_dependencies() when adding a dependency for some of the step targets generated by ExternalProject. See also: http://thread.gmane.org/gmane.comp.programming.tools.cmake.devel/8680/focus=8926 --- Modules/ExternalProject.cmake | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 02383cbb0..851343799 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -303,6 +303,19 @@ line prior to any ``ExternalProject_Add`` calls in your ``CMakeLists.txt`` file:: set_property(DIRECTORY PROPERTY EP_STEP_TARGETS configure build test) + +.. command:: ExternalProject_Add_StepDependencies + + The ``ExternalProject_Add_StepDependencies`` function add some + dependencies for some external project step:: + + ExternalProject_Add_StepDependencies( [target1 [target2 [...]]]) + + This function takes care to set both target and file level + dependencies, and will ensure that parallel builds will not break. + It should be used instead of :command:`add_dependencies()` when adding + a dependency for some of the step targets generated by + ``ExternalProject``. #]=======================================================================] #============================================================================= @@ -1318,6 +1331,7 @@ function(ExternalProject_Add_StepTargets name) _ep_get_step_stampfile(${name} ${step} stamp_file) add_custom_target(${name}-${step} DEPENDS ${stamp_file}) + set_property(TARGET ${name}-${step} PROPERTY _EP_IS_EXTERNAL_PROJECT_STEP 1) set_property(TARGET ${name}-${step} PROPERTY LABELS ${name}) set_property(TARGET ${name}-${step} PROPERTY FOLDER "ExternalProjectTargets/${name}") @@ -1418,6 +1432,7 @@ function(ExternalProject_Add_Step name step) WORKING_DIRECTORY ${work_dir} VERBATIM ) + set_property(TARGET ${name} APPEND PROPERTY _EP_STEPS ${step}) # Add custom "step target"? get_property(step_targets TARGET ${name} PROPERTY _EP_STEP_TARGETS) @@ -1444,6 +1459,49 @@ function(ExternalProject_Add_Step name step) endfunction() +function(ExternalProject_Add_StepDependencies name step) + set(dependencies ${ARGN}) + + # Sanity checks on "name" and "step". + if(NOT TARGET ${name}) + message(FATAL_ERROR "Cannot find target \"${name}\". Perhaps it has not yet been created using ExternalProject_Add.") + endif() + + get_property(is_ep TARGET ${name} PROPERTY _EP_IS_EXTERNAL_PROJECT) + if(NOT is_ep) + message(FATAL_ERROR "Target \"${name}\" was not generated by ExternalProject_Add.") + endif() + + get_property(steps TARGET ${name} PROPERTY _EP_STEPS) + list(FIND steps ${step} is_step) + if(NOT is_step) + message(FATAL_ERROR "External project \"${name}\" does not have a step \"${step}\".") + endif() + + if(TARGET ${name}-${step}) + get_property(is_ep_step TARGET ${name}-${step} PROPERTY _EP_IS_EXTERNAL_PROJECT_STEP) + if(NOT is_ep_step) + message(FATAL_ERROR "Target \"${name}\" was not generated by ExternalProject_Add_StepTargets.") + endif() + endif() + + # Always add file-level dependency, but add target-level dependency + # only if the target exists for that step. + _ep_get_step_stampfile(${name} ${step} stamp_file) + foreach(dep ${dependencies}) + add_custom_command(APPEND + OUTPUT ${stamp_file} + DEPENDS ${dep}) + if(TARGET ${name}-${step}) + foreach(dep ${dependencies}) + add_dependencies(${name}-${step} ${dep}) + endforeach() + endif() + endforeach() + +endfunction() + + function(_ep_add_mkdir_command name) ExternalProject_Get_Property(${name} source_dir binary_dir install_dir stamp_dir download_dir tmp_dir) From 67cfbf8eb7cb2ad0e177b68aa134bc65297929fb Mon Sep 17 00:00:00 2001 From: "Daniele E. Domenichelli" Date: Mon, 3 Nov 2014 16:14:54 +0100 Subject: [PATCH 3/4] ExternalProject: Add unit tests --- Tests/ExternalProjectUpdate/CMakeLists.txt | 3 +- .../Add_StepDependencies.cmake | 20 +++++++++++ .../Add_StepDependencies_no_target.cmake | 10 ++++++ .../ExternalProject/NO_DEPENDS-stderr.txt | 36 +++++++++++++++++++ .../RunCMake/ExternalProject/NO_DEPENDS.cmake | 18 ++++++++++ .../ExternalProject/RunCMakeTest.cmake | 3 ++ 6 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 Tests/RunCMake/ExternalProject/Add_StepDependencies.cmake create mode 100644 Tests/RunCMake/ExternalProject/Add_StepDependencies_no_target.cmake create mode 100644 Tests/RunCMake/ExternalProject/NO_DEPENDS-stderr.txt create mode 100644 Tests/RunCMake/ExternalProject/NO_DEPENDS.cmake diff --git a/Tests/ExternalProjectUpdate/CMakeLists.txt b/Tests/ExternalProjectUpdate/CMakeLists.txt index c33e90b51..582b0a87d 100644 --- a/Tests/ExternalProjectUpdate/CMakeLists.txt +++ b/Tests/ExternalProjectUpdate/CMakeLists.txt @@ -19,6 +19,7 @@ set(base "${CMAKE_BINARY_DIR}/CMakeExternals") set(binary_base "${base}/Build") set_property(DIRECTORY PROPERTY EP_BASE ${base}) set_property(DIRECTORY PROPERTY EP_STEP_TARGETS configure build test) +set_property(DIRECTORY PROPERTY EP_INDEPENDENT_STEP_TARGETS update) set(do_git_tests 0) @@ -68,8 +69,8 @@ if(do_git_tests) CMAKE_GENERATOR "${CMAKE_GENERATOR}" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= INSTALL_COMMAND "" - DEPENDS "SetupLocalGITRepository" ) + ExternalProject_Add_StepDependencies(${proj} download SetupLocalGITRepository) set_property(TARGET ${proj} PROPERTY FOLDER "GIT") endif() diff --git a/Tests/RunCMake/ExternalProject/Add_StepDependencies.cmake b/Tests/RunCMake/ExternalProject/Add_StepDependencies.cmake new file mode 100644 index 000000000..38683f12e --- /dev/null +++ b/Tests/RunCMake/ExternalProject/Add_StepDependencies.cmake @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +include(ExternalProject) + +ExternalProject_Add(BAR URL https://cmake.org/bar.tar.gz) + +ExternalProject_Add(FOO URL https://cmake.org/foo.tar.gz STEP_TARGETS update) +ExternalProject_Add_Step(FOO do_something COMMAND ${CMAKE_COMMAND} -E echo "Doing something") +ExternalProject_Add_Step(FOO do_something_else COMMAND ${CMAKE_COMMAND} -E echo "Doing something else") +ExternalProject_Add_StepTargets(FOO do_something) + +# download and do_something_else are not targets, but the file-level +# dependency are set. +ExternalProject_Add_StepDependencies(FOO download BAR) +ExternalProject_Add_StepDependencies(FOO do_something_else BAR) + +# update and do_something are targets, therefore both file-level and +# target-level dependencies are set. +ExternalProject_Add_StepDependencies(FOO update BAR) +ExternalProject_Add_StepDependencies(FOO do_something BAR) diff --git a/Tests/RunCMake/ExternalProject/Add_StepDependencies_no_target.cmake b/Tests/RunCMake/ExternalProject/Add_StepDependencies_no_target.cmake new file mode 100644 index 000000000..264c3f04c --- /dev/null +++ b/Tests/RunCMake/ExternalProject/Add_StepDependencies_no_target.cmake @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +include(ExternalProject) + +ExternalProject_Add(BAR URL https://cmake.org/bar.tar.gz) + +ExternalProject_Add(FOO URL https://cmake.org/foo.tar.gz STEP_TARGETS update) +ExternalProject_Add_Step(FOO do_something COMMAND ${CMAKE_COMMAND} -E echo "Doing something") +ExternalProject_Add_Step(FOO do_something_else COMMAND ${CMAKE_COMMAND} -E echo "Doing something else") +ExternalProject_Add_StepTargets(FOO do_something) diff --git a/Tests/RunCMake/ExternalProject/NO_DEPENDS-stderr.txt b/Tests/RunCMake/ExternalProject/NO_DEPENDS-stderr.txt new file mode 100644 index 000000000..4cb051dbb --- /dev/null +++ b/Tests/RunCMake/ExternalProject/NO_DEPENDS-stderr.txt @@ -0,0 +1,36 @@ +CMake Warning \(dev\) at .*/Modules/ExternalProject.cmake:[0-9]+. \(message\): + Using NO_DEPENDS for "configure" step might break parallel builds +Call Stack \(most recent call first\): + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_StepTargets\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_configure_command\) + NO_DEPENDS.cmake:[0-9]+ \(ExternalProject_Add\) + CMakeLists.txt:[0-9]+ \(include\) +This warning is for project developers. Use -Wno-dev to suppress it. + +CMake Warning \(dev\) at .*/Modules/ExternalProject.cmake:[0-9]+. \(message\): + Using NO_DEPENDS for "build" step might break parallel builds +Call Stack \(most recent call first\): + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_StepTargets\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_build_command\) + NO_DEPENDS.cmake:[0-9]+ \(ExternalProject_Add\) + CMakeLists.txt:[0-9]+ \(include\) +This warning is for project developers. Use -Wno-dev to suppress it. + +CMake Warning \(dev\) at .*/Modules/ExternalProject.cmake:[0-9]+. \(message\): + Using NO_DEPENDS for "install" step might break parallel builds +Call Stack \(most recent call first\): + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_StepTargets\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\) + .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_install_command\) + NO_DEPENDS.cmake:[0-9]+ \(ExternalProject_Add\) + CMakeLists.txt:[0-9]+ \(include\) +This warning is for project developers. Use -Wno-dev to suppress it. + +CMake Warning \(dev\) at .*/Modules/ExternalProject.cmake:[0-9]+. \(message\): + Using NO_DEPENDS for "test" step might break parallel builds +Call Stack \(most recent call first\): + NO_DEPENDS.cmake:[0-9]+ \(ExternalProject_Add_StepTargets\) + CMakeLists.txt:[0-9]+ \(include\) +This warning is for project developers. Use -Wno-dev to suppress it. diff --git a/Tests/RunCMake/ExternalProject/NO_DEPENDS.cmake b/Tests/RunCMake/ExternalProject/NO_DEPENDS.cmake new file mode 100644 index 000000000..57626d679 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/NO_DEPENDS.cmake @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 2.8.12) + +include(ExternalProject RESULT_VARIABLE GOO) + +set_property(DIRECTORY PROPERTY EP_INDEPENDENT_STEP_TARGETS download patch update configure build) + +ExternalProject_Add(FOO + URL https://example.org/foo.tar.gz) + +ExternalProject_Add(BAR + URL https://example.org/bar.tar.gz + TEST_COMMAND echo test + INDEPENDENT_STEP_TARGETS install) +# This one should not give a warning +ExternalProject_Add_Step(BAR bar + COMMAND echo bar) + +ExternalProject_Add_StepTargets(BAR NO_DEPENDS test bar) diff --git a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake index 1614eccaf..0f5dcefb7 100644 --- a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake @@ -3,3 +3,6 @@ include(RunCMake) run_cmake(CMAKE_CACHE_ARGS) run_cmake(CMAKE_CACHE_DEFAULT_ARGS) run_cmake(CMAKE_CACHE_mix) +run_cmake(NO_DEPENDS) +run_cmake(Add_StepDependencies) +run_cmake(Add_StepDependencies_no_target) From 468fb734b4462c2c338abff8140f90a3c9d213a5 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 4 Nov 2014 12:56:14 -0500 Subject: [PATCH 4/4] Help: Add notes for topic 'ExternalProject_independent-step-targets' --- .../dev/ExternalProject_independent-step-targets.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Help/release/dev/ExternalProject_independent-step-targets.rst diff --git a/Help/release/dev/ExternalProject_independent-step-targets.rst b/Help/release/dev/ExternalProject_independent-step-targets.rst new file mode 100644 index 000000000..02e8db838 --- /dev/null +++ b/Help/release/dev/ExternalProject_independent-step-targets.rst @@ -0,0 +1,6 @@ +ExternalProject_independent-step-targets +---------------------------------------- + +* The :module:`ExternalProject` module learned options to create + independent external project step targets that do not depend + on the builtin steps.