CTest: Stop telling 'make' to ignore errors with -i

Add policy CMP0061 to maintain compatibility for existing projects.

Help/command/build_command.rst

@@ -19,7 +19,8 @@ Sets the given ``<variable>`` to a command-line string of the form::
 where ``<cmake>`` is the location of the :manual:`cmake(1)` command-line
 tool, and ``<config>`` and ``<target>`` are the values provided to the
 ``CONFIGURATION`` and ``TARGET`` options, if any.  The trailing ``-- -i``
-option is added for Makefile generators.
+option is added for :ref:`Makefile Generators` if policy :policy:`CMP0061`
+is not set to ``NEW``.
 
 When invoked, this ``cmake --build`` command line will launch the
 underlying build system tool.

Help/manual/cmake-policies.7.rst

@@ -118,3 +118,4 @@ All Policies
    /policy/CMP0058
    /policy/CMP0059
    /policy/CMP0060
+   /policy/CMP0061

Help/policy/CMP0061.rst

@@ -0,0 +1,24 @@
+CMP0061
+-------
+
+CTest does not by default tell ``make`` to ignore errors (``-i``).
+
+The :command:`ctest_build` and :command:`build_command` commands no
+longer generate build commands for :ref:`Makefile Generators` with
+the ``-i`` option.  Previously this was done to help build as much
+of tested projects as possible.  However, this behavior is not
+consistent with other generators and also causes the return code
+of the ``make`` tool to be meaningless.
+
+Of course users may still add this option manually by setting
+:variable:`CTEST_BUILD_COMMAND` or the ``MAKECOMMAND`` cache entry.
+See the :ref:`CTest Build Step` ``MakeCommand`` setting documentation
+for their effects.
+
+The ``OLD`` behavior for this policy is to add ``-i`` to ``make``
+calls in CTest.  The ``NEW`` behavior for this policy is to not
+add ``-i``.
+
+This policy was introduced in CMake version 3.3.  Unlike most policies,
+CMake version |release| does *not* warn when this policy is not set and
+simply uses OLD behavior.

Help/release/dev/ctest-no-make-i.rst

@@ -0,0 +1,7 @@
+ctest-no-make-i
+---------------
+
+* The :command:`ctest_build` and :command:`build_command` commands
+  no longer tell ``make`` tools to ignore errors with the ``-i`` option.
+  Previously this was done for :ref:`Makefile Generators` but not others.
+  See policy :policy:`CMP0061`.

Source/CTest/cmCTestBuildCommand.cxx

@@ -141,7 +141,8 @@ cmCTestGenericHandler* cmCTestBuildCommand::InitializeHandler()
         = this->GlobalGenerator->
         GenerateCMakeBuildCommand(cmakeBuildTarget ? cmakeBuildTarget : "",
           cmakeBuildConfiguration,
-          cmakeBuildAdditionalFlags ? cmakeBuildAdditionalFlags : "", true);
+          cmakeBuildAdditionalFlags ? cmakeBuildAdditionalFlags : "",
+          this->Makefile->IgnoreErrorsCMP0061());
       cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
         "SetMakeCommand:" << buildCommand << "\n", this->Quiet);
       this->CTest->SetCTestConfiguration("MakeCommand", buildCommand.c_str(),

Source/cmBuildCommand.cxx

@@ -106,7 +106,8 @@ bool cmBuildCommand
     }
 
   std::string makecommand = this->Makefile->GetGlobalGenerator()
-      ->GenerateCMakeBuildCommand(target, configuration, "", true);
+    ->GenerateCMakeBuildCommand(target, configuration, "",
+                                this->Makefile->IgnoreErrorsCMP0061());
 
   this->Makefile->AddDefinition(variable, makecommand.c_str());
 
@@ -135,7 +136,8 @@ bool cmBuildCommand
     }
 
   std::string makecommand = this->Makefile->GetGlobalGenerator()
-      ->GenerateCMakeBuildCommand("", configType, "", true);
+    ->GenerateCMakeBuildCommand("", configType, "",
+                                this->Makefile->IgnoreErrorsCMP0061());
 
   if(cacheValue)
     {

Source/cmBuildCommand.h

@@ -53,6 +53,8 @@ public:
   virtual std::string GetName() const {return "build_command";}
 
   cmTypeMacro(cmBuildCommand, cmCommand);
+private:
+  bool IgnoreErrors() const;
 };
 
 #endif

Source/cmMakefile.cxx

@@ -4942,6 +4942,26 @@ void cmMakefile::RecordPolicies(cmPolicies::PolicyMap& pm)
     }
 }
 
+//----------------------------------------------------------------------------
+bool cmMakefile::IgnoreErrorsCMP0061() const
+{
+  bool ignoreErrors = true;
+  switch (this->GetPolicyStatus(cmPolicies::CMP0061))
+    {
+    case cmPolicies::WARN:
+      // No warning for this policy!
+    case cmPolicies::OLD:
+      break;
+    case cmPolicies::REQUIRED_IF_USED:
+    case cmPolicies::REQUIRED_ALWAYS:
+    case cmPolicies::NEW:
+      ignoreErrors = false;
+      break;
+    }
+  return ignoreErrors;
+}
+
+//----------------------------------------------------------------------------
 #define FEATURE_STRING(F) , #F
 static const char * const C_FEATURES[] = {
   0

Source/cmMakefile.h

@@ -415,6 +415,8 @@ public:
   bool HasCMP0054AlreadyBeenReported(
     cmListFileContext context) const;
 
+  bool IgnoreErrorsCMP0061() const;
+
   const char* GetHomeDirectory() const;
   const char* GetHomeOutputDirectory() const;
 

Source/cmPolicies.h

@@ -208,6 +208,9 @@ class cmPolicy;
     3, 3, 0, cmPolicies::WARN) \
   SELECT(POLICY, CMP0060, \
     "Link libraries by full path even in implicit directories.", \
+    3, 3, 0, cmPolicies::WARN) \
+  SELECT(POLICY, CMP0061, \
+    "CTest does not by default tell make to ignore errors (-i).", \
     3, 3, 0, cmPolicies::WARN)
 
 #define CM_SELECT_ID(F, A1, A2, A3, A4, A5, A6) F(A1)

Tests/RunCMake/build_command/CMP0061-NEW-stderr.txt

@@ -0,0 +1,10 @@
+^[^
+]+ --build \. --config "Release"
+[^
+]+ --build \. --config "Release" --target "MyTarget"
+[^
+]+ --build \. --config "Debug"
+[^
+]+ --build \. --config "Debug" --target "MyTarget"
+[^
+]+ --build \. --config "Release"$

Tests/RunCMake/build_command/CMP0061-NEW.cmake

@@ -0,0 +1,2 @@
+cmake_policy(SET CMP0061 NEW)
+include(CMP0061Common.cmake)

Tests/RunCMake/build_command/CMP0061-OLD-make-stderr.txt

@@ -0,0 +1,10 @@
+^[^
+]+ --build \. --config "Release" -- -i
+[^
+]+ --build \. --config "Release" --target "MyTarget" -- -i
+[^
+]+ --build \. --config "Debug" -- -i
+[^
+]+ --build \. --config "Debug" --target "MyTarget" -- -i
+[^
+]+ --build \. --config "Release" -- -i$

Tests/RunCMake/build_command/CMP0061-OLD-make.cmake

@@ -0,0 +1,2 @@
+cmake_policy(SET CMP0061 OLD)
+include(CMP0061Common.cmake)

Tests/RunCMake/build_command/CMP0061-OLD-other-stderr.txt

@@ -0,0 +1,10 @@
+^[^
+]+ --build \. --config "Release"
+[^
+]+ --build \. --config "Release" --target "MyTarget"
+[^
+]+ --build \. --config "Debug"
+[^
+]+ --build \. --config "Debug" --target "MyTarget"
+[^
+]+ --build \. --config "Release"$

Tests/RunCMake/build_command/CMP0061-OLD-other.cmake

@@ -0,0 +1,2 @@
+cmake_policy(SET CMP0061 OLD)
+include(CMP0061Common.cmake)

Tests/RunCMake/build_command/CMP0061Common.cmake

@@ -0,0 +1,10 @@
+build_command(command)
+message("${command}")
+build_command(command TARGET MyTarget)
+message("${command}")
+build_command(command CONFIGURATION Debug)
+message("${command}")
+build_command(command CONFIGURATION Debug TARGET MyTarget)
+message("${command}")
+build_command(cache_command "${CMAKE_MAKE_PROGRAM}")
+message("${cache_command}")

Tests/RunCMake/build_command/RunCMakeTest.cmake

@@ -1,4 +1,5 @@
 include(RunCMake)
+unset(ENV{CMAKE_CONFIG_TYPE})
 
 run_cmake(ErrorsOFF)
 run_cmake(ErrorsON)
@@ -6,3 +7,10 @@ run_cmake(ErrorsON)
 set(RunCMake_TEST_OPTIONS -DNoProject=1)
 run_cmake(BeforeProject)
 unset(RunCMake_TEST_OPTIONS)
+
+run_cmake(CMP0061-NEW)
+if(RunCMake_GENERATOR MATCHES "Make")
+  run_cmake(CMP0061-OLD-make)
+else()
+  run_cmake(CMP0061-OLD-other)
+endif()

Tests/RunCMake/ctest_build/BuildFailure-CMP0061-OLD-result.txt

@@ -0,0 +1 @@
+(0|-1|255)

Tests/RunCMake/ctest_build/BuildFailure-CMP0061-OLD-stderr.txt

@@ -0,0 +1,2 @@
+^(Error\(s\) when building project
+)?ctest_build returned zero$

Tests/RunCMake/ctest_build/BuildFailure-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_build/BuildFailure-stderr.txt

@@ -0,0 +1,2 @@
+^Error\(s\) when building project
+ctest_build returned non-zero$

Tests/RunCMake/ctest_build/CMakeLists.txt.in

@@ -2,3 +2,4 @@ cmake_minimum_required(VERSION 3.1)
 project(CTestBuild@CASE_NAME@ NONE)
 include(CTest)
 add_test(NAME RunCMakeVersion COMMAND "${CMAKE_COMMAND}" --version)
+@CASE_CMAKELISTS_SUFFIX_CODE@

Tests/RunCMake/ctest_build/RunCMakeTest.cmake

@@ -8,3 +8,26 @@ function(run_ctest_build CASE_NAME)
 endfunction()
 
 run_ctest_build(BuildQuiet QUIET)
+
+function(run_BuildFailure)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_custom_target(BuildFailure ALL COMMAND command-does-not-exist)
+]])
+  set(CASE_TEST_PREFIX_CODE [[
+cmake_policy(SET CMP0061 NEW)
+]])
+  set(CASE_TEST_SUFFIX_CODE [[
+if (ctest_build_return_value)
+  message("ctest_build returned non-zero")
+else()
+  message("ctest_build returned zero")
+endif()
+]])
+  run_ctest(BuildFailure)
+
+  if (RunCMake_GENERATOR MATCHES "Makefiles")
+    set(CASE_TEST_PREFIX_CODE "")
+    run_ctest(BuildFailure-CMP0061-OLD)
+  endif()
+endfunction()
+run_BuildFailure()

Tests/RunCMake/ctest_build/test.cmake.in

@@ -1,4 +1,5 @@
 cmake_minimum_required(VERSION 3.1)
+@CASE_TEST_PREFIX_CODE@
 
 set(CTEST_SITE                          "test-site")
 set(CTEST_BUILD_NAME                    "test-build-name")
@@ -12,4 +13,5 @@ set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
 set(ctest_build_args "@CASE_CTEST_BUILD_ARGS@")
 ctest_start(Experimental)
 ctest_configure()
-ctest_build(${ctest_build_args})
+ctest_build(${ctest_build_args} RETURN_VALUE ctest_build_return_value)
+@CASE_TEST_SUFFIX_CODE@