Honor visibility properties for all target types (#15556)

The <LANG>_VISIBILITY_PRESET and VISIBILITY_INLINES_HIDDEN were first
merged in commit v2.8.12~322 (Merge topic 'VISIBILITY_PRESET-property',
2013-06-05) but worked only for shared libraries and executables with
exports.  Prior to commit v3.0.0-rc1~581^2 (GenerateExportHeader:
Deprecate add_compiler_export_flags function., 2013-09-02) the
add_compiler_export_flags function was used to add visibility flags to
all targets.

The visibility flags are useful for sources in all target types because
they may be later linked into shared libraries or executables with
exports.  Introduce policy CMP0063 to enable them for all target types
while preserving compatibility with existing projects that do not expect
this.

Help/manual/cmake-policies.7.rst

@@ -120,3 +120,4 @@ All Policies
    /policy/CMP0060
    /policy/CMP0061
    /policy/CMP0062
+   /policy/CMP0063

Help/policy/CMP0063.rst

@@ -0,0 +1,26 @@
+CMP0063
+-------
+
+Honor visibility properties for all target types.
+
+The :prop_tgt:`<LANG>_VISIBILITY_PRESET` and
+:prop_tgt:`VISIBILITY_INLINES_HIDDEN` target properties affect visibility
+of symbols during dynamic linking.  When first introduced these properties
+affected compilation of sources only in shared libraries, module libraries,
+and executables with the :prop_tgt:`ENABLE_EXPORTS` property set.  This
+was sufficient for the basic use cases of shared libraries and executables
+with plugins.  However, some sources may be compiled as part of static
+libraries or object libraries and then linked into a shared library later.
+CMake 3.3 and above prefer to honor these properties for sources compiled
+in all target types.  This policy preserves compatibility for projects
+expecting the properties to work only for some target types.
+
+The ``OLD`` behavior for this policy is to ignore the visibility properties
+for static libraries, object libraries, and executables without exports.
+The ``NEW`` behavior for this policy is to honor the visibility properties
+for all target types.
+
+This policy was introduced in CMake version 3.3.  CMake version
+|release| warns when the policy is not set and uses ``OLD`` behavior.  Use
+the :command:`cmake_policy()` command to set it to ``OLD`` or ``NEW``
+explicitly.

Help/prop_tgt/LANG_VISIBILITY_PRESET.rst

@@ -5,7 +5,9 @@ Value for symbol visibility compile flags
 
 The ``<LANG>_VISIBILITY_PRESET`` property determines the value passed in a
 visibility related compile option, such as ``-fvisibility=`` for ``<LANG>``.
-This property only has an affect for libraries and executables with
-exports.  This property is initialized by the value of the
+This property affects compilation in sources of all types of targets
+(subject to policy :policy:`CMP0063`).
+
+This property is initialized by the value of the
 :variable:`CMAKE_<LANG>_VISIBILITY_PRESET` variable if it is set when a
 target is created.

Help/prop_tgt/VISIBILITY_INLINES_HIDDEN.rst

@@ -5,7 +5,9 @@ Whether to add a compile flag to hide symbols of inline functions
 
 The ``VISIBILITY_INLINES_HIDDEN`` property determines whether a flag for
 hiding symbols for inline functions, such as ``-fvisibility-inlines-hidden``,
-should be used when invoking the compiler.  This property only has an affect
-for libraries and executables with exports.  This property is initialized by
+should be used when invoking the compiler.  This property affects compilation
+in sources of all types of targets (subject to policy :policy:`CMP0063`).
+
+This property is initialized by
 the value of the :variable:`CMAKE_VISIBILITY_INLINES_HIDDEN` variable if it
 is set when a target is created.

Help/release/dev/extend-visibility-properties.rst

@@ -0,0 +1,7 @@
+extend-visibility-properties
+----------------------------
+
+* The :prop_tgt:`<LANG>_VISIBILITY_PRESET` and
+  :prop_tgt:`VISIBILITY_INLINES_HIDDEN` target properties now
+  affect compilation in sources of all target types.  See
+  policy :policy:`CMP0063`.

Source/cmLocalGenerator.cxx

@@ -2248,7 +2248,8 @@ AddCompilerRequirementFlag(std::string &flags, cmTarget const* target,
 static void AddVisibilityCompileOption(std::string &flags,
                                        cmTarget const* target,
                                        cmLocalGenerator *lg,
-                                       const std::string& lang)
+                                       const std::string& lang,
+                                       std::string* warnCMP0063)
 {
   std::string l(lang);
   std::string compileOption = "CMAKE_" + l + "_COMPILE_OPTIONS_VISIBILITY";
@@ -2264,6 +2265,11 @@ static void AddVisibilityCompileOption(std::string &flags,
     {
     return;
     }
+  if (warnCMP0063)
+    {
+    *warnCMP0063 += "  " + flagDefine + "\n";
+    return;
+    }
   if (strcmp(prop, "hidden") != 0
       && strcmp(prop, "default") != 0
       && strcmp(prop, "protected") != 0
@@ -2281,7 +2287,8 @@ static void AddVisibilityCompileOption(std::string &flags,
 
 static void AddInlineVisibilityCompileOption(std::string &flags,
                                        cmTarget const* target,
-                                       cmLocalGenerator *lg)
+                                       cmLocalGenerator *lg,
+                                       std::string* warnCMP0063)
 {
   std::string compileOption
                 = "CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN";
@@ -2296,6 +2303,11 @@ static void AddInlineVisibilityCompileOption(std::string &flags,
     {
     return;
     }
+  if (warnCMP0063)
+    {
+    *warnCMP0063 += "  VISIBILITY_INLINES_HIDDEN\n";
+    return;
+    }
   lg->AppendFlags(flags, opt);
 }
 
@@ -2304,25 +2316,49 @@ void cmLocalGenerator
 ::AddVisibilityPresetFlags(std::string &flags, cmTarget const* target,
                             const std::string& lang)
 {
-  int targetType = target->GetType();
-  bool suitableTarget = ((targetType == cmTarget::SHARED_LIBRARY)
-                      || (targetType == cmTarget::MODULE_LIBRARY)
-                      || (target->IsExecutableWithExports()));
-
-  if (!suitableTarget)
-    {
-    return;
-    }
-
   if (lang.empty())
     {
     return;
     }
-  AddVisibilityCompileOption(flags, target, this, lang);
+
+  std::string warnCMP0063;
+  std::string *pWarnCMP0063 = 0;
+  if (target->GetType() != cmTarget::SHARED_LIBRARY &&
+      target->GetType() != cmTarget::MODULE_LIBRARY &&
+      !target->IsExecutableWithExports())
+    {
+    switch (target->GetPolicyStatusCMP0063())
+      {
+      case cmPolicies::OLD:
+        return;
+      case cmPolicies::WARN:
+        pWarnCMP0063 = &warnCMP0063;
+        break;
+      default:
+        break;
+      }
+    }
+
+  AddVisibilityCompileOption(flags, target, this, lang, pWarnCMP0063);
 
   if(lang == "CXX")
     {
-    AddInlineVisibilityCompileOption(flags, target, this);
+    AddInlineVisibilityCompileOption(flags, target, this, pWarnCMP0063);
+    }
+
+  if (!warnCMP0063.empty() &&
+      this->WarnCMP0063.insert(target).second)
+    {
+    std::ostringstream w;
+    w <<
+      cmPolicies::GetPolicyWarning(cmPolicies::CMP0063) << "\n"
+      "Target \"" << target->GetName() << "\" of "
+      "type \"" << cmTarget::GetTargetTypeName(target->GetType()) << "\" "
+      "has the following visibility properties set for " << lang << ":\n" <<
+      warnCMP0063 <<
+      "For compatibility CMake is not honoring them for this target.";
+    target->GetMakefile()->GetCMakeInstance()
+      ->IssueMessage(cmake::AUTHOR_WARNING, w.str(), target->GetBacktrace());
     }
 }
 

Source/cmLocalGenerator.h

@@ -456,6 +456,8 @@ protected:
   std::string::size_type ObjectPathMax;
   std::set<std::string> ObjectMaxPathViolations;
 
+  std::set<cmTarget const*> WarnCMP0063;
+
   bool LinkScriptShell;
   bool UseRelativePaths;
   bool Configured;

Source/cmPolicies.h

@@ -214,6 +214,9 @@ class cmPolicy;
     3, 3, 0, cmPolicies::WARN) \
   SELECT(POLICY, CMP0062, \
     "Disallow install() of export() result.", \
+    3, 3, 0, cmPolicies::WARN) \
+  SELECT(POLICY, CMP0063, \
+    "Honor visibility properties for all target types.", \
     3, 3, 0, cmPolicies::WARN)
 
 #define CM_SELECT_ID(F, A1, A2, A3, A4, A5, A6) F(A1)

Source/cmTarget.h

@@ -39,7 +39,8 @@
   F(CMP0042) \
   F(CMP0046) \
   F(CMP0052) \
-  F(CMP0060)
+  F(CMP0060) \
+  F(CMP0063)
 
 class cmake;
 class cmMakefile;

Tests/RunCMake/TargetPolicies/PolicyList-stderr.txt

@@ -18,6 +18,7 @@
    \* CMP0046
    \* CMP0052
    \* CMP0060
+   \* CMP0063
 
 Call Stack \(most recent call first\):
   CMakeLists.txt:3 \(include\)

Tests/RunCMake/VisibilityPreset/CMP0063-Common.cmake

@@ -0,0 +1,7 @@
+set(CMAKE_VISIBILITY_INLINES_HIDDEN 1)
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+add_executable(myexe lib.cpp)
+add_library(mystatic STATIC lib.cpp)
+add_library(myshared SHARED lib.cpp)
+add_library(mymodule MODULE lib.cpp)
+add_library(myobject OBJECT lib.cpp)

Tests/RunCMake/VisibilityPreset/CMP0063-NEW.cmake

@@ -0,0 +1,8 @@
+cmake_policy(SET CMP0063 NEW)
+enable_language(CXX)
+
+# Ensure CMake would warn even if toolchain does not really have these flags.
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN "-fvisibility-inlines-hidden")
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY "-fvisibility=")
+
+include(CMP0063-Common.cmake)

Tests/RunCMake/VisibilityPreset/CMP0063-OLD.cmake

@@ -0,0 +1,8 @@
+cmake_policy(SET CMP0063 OLD)
+enable_language(CXX)
+
+# Ensure CMake would warn even if toolchain does not really have these flags.
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN "-fvisibility-inlines-hidden")
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY "-fvisibility=")
+
+include(CMP0063-Common.cmake)

Tests/RunCMake/VisibilityPreset/CMP0063-WARN-no.cmake

@@ -0,0 +1,8 @@
+
+enable_language(CXX)
+
+# Ensure CMake does not warn even if toolchain really does have these flags.
+unset(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN)
+unset(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY)
+
+include(CMP0063-Common.cmake)

Tests/RunCMake/VisibilityPreset/CMP0063-WARN-yes-stderr.txt

@@ -0,0 +1,50 @@
+^CMake Warning \(dev\) at CMP0063-Common.cmake:[0-9]+ \(add_executable\):
+  Policy CMP0063 is not set: Honor visibility properties for all target
+  types.  Run "cmake --help-policy CMP0063" for policy details.  Use the
+  cmake_policy command to set the policy and suppress this warning.
+
+  Target "myexe" of type "EXECUTABLE" has the following visibility properties
+  set for CXX:
+
+    CXX_VISIBILITY_PRESET
+    VISIBILITY_INLINES_HIDDEN
+
+  For compatibility CMake is not honoring them for this target.
+Call Stack \(most recent call first\):
+  CMP0063-WARN-yes.cmake:[0-9]+ \(include\)
+  CMakeLists.txt:[0-9]+ \(include\)
+This warning is for project developers.  Use -Wno-dev to suppress it.
++
+CMake Warning \(dev\) at CMP0063-Common.cmake:[0-9]+ \(add_library\):
+  Policy CMP0063 is not set: Honor visibility properties for all target
+  types.  Run "cmake --help-policy CMP0063" for policy details.  Use the
+  cmake_policy command to set the policy and suppress this warning.
+
+  Target "myobject" of type "OBJECT_LIBRARY" has the following visibility
+  properties set for CXX:
+
+    CXX_VISIBILITY_PRESET
+    VISIBILITY_INLINES_HIDDEN
+
+  For compatibility CMake is not honoring them for this target.
+Call Stack \(most recent call first\):
+  CMP0063-WARN-yes.cmake:[0-9]+ \(include\)
+  CMakeLists.txt:[0-9]+ \(include\)
+This warning is for project developers.  Use -Wno-dev to suppress it.
++
+CMake Warning \(dev\) at CMP0063-Common.cmake:[0-9]+ \(add_library\):
+  Policy CMP0063 is not set: Honor visibility properties for all target
+  types.  Run "cmake --help-policy CMP0063" for policy details.  Use the
+  cmake_policy command to set the policy and suppress this warning.
+
+  Target "mystatic" of type "STATIC_LIBRARY" has the following visibility
+  properties set for CXX:
+
+    CXX_VISIBILITY_PRESET
+    VISIBILITY_INLINES_HIDDEN
+
+  For compatibility CMake is not honoring them for this target.
+Call Stack \(most recent call first\):
+  CMP0063-WARN-yes.cmake:[0-9]+ \(include\)
+  CMakeLists.txt:[0-9]+ \(include\)
+This warning is for project developers.  Use -Wno-dev to suppress it.$

Tests/RunCMake/VisibilityPreset/CMP0063-WARN-yes.cmake

@@ -0,0 +1,8 @@
+
+enable_language(CXX)
+
+# Ensure CMake warns even if toolchain does not really have these flags.
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN "-fvisibility-inlines-hidden")
+set(CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY "-fvisibility=")
+
+include(CMP0063-Common.cmake)

Tests/RunCMake/VisibilityPreset/CMakeLists.txt

@@ -1,3 +1,3 @@
-cmake_minimum_required(VERSION 2.8.4)
+cmake_minimum_required(VERSION 3.2)
 project(${RunCMake_TEST} NONE)
 include(${RunCMake_TEST}.cmake)

Tests/RunCMake/VisibilityPreset/RunCMakeTest.cmake

@@ -1,3 +1,7 @@
 include(RunCMake)
 
 run_cmake(PropertyTypo)
+run_cmake(CMP0063-OLD)
+run_cmake(CMP0063-WARN-yes)
+run_cmake(CMP0063-WARN-no)
+run_cmake(CMP0063-NEW)

Tests/Visibility/CMakeLists.txt

@@ -1,13 +1,66 @@
 cmake_minimum_required(VERSION 3.2)
+cmake_policy(SET CMP0063 NEW)
+
 project(Visibility)
 
-add_library(inlines_hidden SHARED foo.cpp bar.c)
-set_property(TARGET inlines_hidden PROPERTY VISIBILITY_INLINES_HIDDEN ON)
-target_compile_options(inlines_hidden PRIVATE -Werror)
+add_library(hidden1 SHARED hidden.c)
+set_property(TARGET hidden1 PROPERTY C_VISIBILITY_PRESET hidden)
 
-add_custom_command(TARGET inlines_hidden POST_BUILD
-  COMMAND ${CMAKE_COMMAND}
+add_library(hidden_object OBJECT hidden.c)
+set_property(TARGET hidden_object PROPERTY C_VISIBILITY_PRESET hidden)
+set_property(TARGET hidden_object PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+add_library(hidden_static STATIC hidden.c)
+set_property(TARGET hidden_static PROPERTY C_VISIBILITY_PRESET hidden)
+set_property(TARGET hidden_static PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+add_library(hidden2 SHARED $<TARGET_OBJECTS:hidden_object> shared.c)
+
+add_library(hidden3 SHARED shared.c)
+target_link_libraries(hidden3 hidden_static)
+
+foreach(t
+    hidden1
+    hidden2
+    hidden3
+    )
+  add_custom_command(TARGET ${t} POST_BUILD
+    COMMAND ${CMAKE_COMMAND}
     -DCMAKE_NM=${CMAKE_NM}
-    -DTEST_LIBRARY_PATH=$<TARGET_FILE:inlines_hidden>
+    -DTEST_LIBRARY_PATH=$<TARGET_FILE:${t}>
     -P ${CMAKE_CURRENT_SOURCE_DIR}/verify.cmake
-)
+    )
+endforeach()
+
+
+add_library(inlines_hidden1 SHARED foo.cpp bar.c)
+set_property(TARGET inlines_hidden1 PROPERTY VISIBILITY_INLINES_HIDDEN ON)
+target_compile_options(inlines_hidden1 PRIVATE -Werror)
+
+add_library(inlines_hidden_object OBJECT foo.cpp bar.c)
+set_property(TARGET inlines_hidden_object PROPERTY VISIBILITY_INLINES_HIDDEN ON)
+set_property(TARGET inlines_hidden_object PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_compile_options(inlines_hidden_object PRIVATE -Werror)
+
+add_library(inlines_hidden_static STATIC foo.cpp bar.c)
+set_property(TARGET inlines_hidden_static PROPERTY VISIBILITY_INLINES_HIDDEN ON)
+set_property(TARGET inlines_hidden_static PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_compile_options(inlines_hidden_static PRIVATE -Werror)
+
+add_library(inlines_hidden2 SHARED $<TARGET_OBJECTS:inlines_hidden_object> shared.cpp)
+
+add_library(inlines_hidden3 SHARED shared.cpp)
+target_link_libraries(inlines_hidden3 inlines_hidden_static)
+
+foreach(t
+    inlines_hidden1
+    inlines_hidden2
+    inlines_hidden3
+    )
+  add_custom_command(TARGET ${t} POST_BUILD
+    COMMAND ${CMAKE_COMMAND}
+    -DCMAKE_NM=${CMAKE_NM}
+    -DTEST_LIBRARY_PATH=$<TARGET_FILE:${t}>
+    -P ${CMAKE_CURRENT_SOURCE_DIR}/verify.cmake
+    )
+endforeach()

Tests/Visibility/hidden.c

@@ -0,0 +1,4 @@
+int hidden_function(void) { return 0; }
+
+__attribute__((visibility("default")))
+int not_hidden(void) { return hidden_function(); }

Tests/Visibility/shared.c

@@ -0,0 +1,3 @@
+extern int not_hidden(void);
+
+int shared(void) { return not_hidden(); }

Tests/Visibility/shared.cpp

@@ -0,0 +1,8 @@
+extern "C" int bar(void);
+void baz();
+
+int shared()
+{
+  baz();
+  return bar();
+}

Tests/Visibility/verify.cmake

@@ -8,7 +8,7 @@ if(NOT "${RESULT}" STREQUAL "0")
   message(FATAL_ERROR "nm failed [${RESULT}] [${OUTPUT}] [${ERROR}]")
 endif()
 
-if(${OUTPUT} MATCHES "Foo[^\\n]*bar")
+if(${OUTPUT} MATCHES "(Foo[^\\n]*bar|hidden_function)")
   message(FATAL_ERROR
-    "Found Foo::bar() which should have been hidden [${OUTPUT}]")
+    "Found ${CMAKE_MATCH_1} which should have been hidden [${OUTPUT}]")
 endif()