ENH: Added computation of object file names that are almost always short enough to not exceed the filesystem path length limitation. This is useful when a source file from outside the tree is referenced with a long full path. The object file name previously would contain the entire path which when combined with the build output directory could exceed the filesystem limit. Now CMake recognizes this case and replaces enough of the beginning of the full path to the source file with an md5sum of the replaced portion to make the name fit on disk. This addresses bug#4520.

Source/cmLocalGenerator.cxx

@@ -28,6 +28,11 @@
 #include "cmTest.h"
 #include "cmake.h"
 
+#if defined(CMAKE_BUILD_WITH_CMAKE)
+# define CM_LG_ENCODE_OBJECT_NAMES
+# include <cmsys/MD5.h>
+#endif
+
 #include <cmsys/System.h>
 
 #include <ctype.h> // for isalpha
@@ -2376,8 +2381,81 @@ cmLocalGenerator
     }
 }
 
+#if defined(CM_LG_ENCODE_OBJECT_NAMES)
+static std::string cmLocalGeneratorMD5(const char* input)
+{
+  char md5out[32];
+  cmsysMD5* md5 = cmsysMD5_New();
+  cmsysMD5_Initialize(md5);
+  cmsysMD5_Append(md5, reinterpret_cast<unsigned char const*>(input), -1);
+  cmsysMD5_FinalizeHex(md5, md5out);
+  cmsysMD5_Delete(md5);
+  return std::string(md5out, 32);
+}
+
+static bool
+cmLocalGeneratorShortenObjectName(std::string& objName,
+                                  std::string::size_type max_len)
+{
+  // Replace the beginning of the path portion of the object name with
+  // its own md5 sum.
+  std::string::size_type pos = objName.find('/', objName.size()-max_len+32);
+  if(pos != objName.npos)
+    {
+    std::string md5name = cmLocalGeneratorMD5(objName.substr(0, pos).c_str());
+    md5name += objName.substr(pos);
+    objName = md5name;
+
+    // The object name is now short enough.
+    return true;
+    }
+  else
+    {
+    // The object name could not be shortened enough.
+    return false;
+    }
+}
+
+static bool cmLocalGeneratorCheckObjectName(std::string& objName,
+                                            std::string::size_type dir_len)
+{
+  // Choose a maximum file name length.
+#if defined(_WIN32) || defined(__CYGWIN__)
+  std::string::size_type const max_total_len = 250;
+#else
+  std::string::size_type const max_total_len = 1000;
+#endif
+
+  // Enforce the maximum file name length if possible.
+  std::string::size_type max_obj_len = max_total_len;
+  if(dir_len < max_total_len)
+    {
+    max_obj_len = max_total_len - dir_len;
+    if(objName.size() > max_obj_len)
+      {
+      // The current object file name is too long.  Try to shorten it.
+      return cmLocalGeneratorShortenObjectName(objName, max_obj_len);
+      }
+    else
+      {
+      // The object file name is short enough.
+      return true;
+      }
+    }
+  else
+    {
+    // The build directory in which the object will be stored is
+    // already too deep.
+    return false;
+    }
+}
+#endif
+
 //----------------------------------------------------------------------------
-std::string& cmLocalGenerator::CreateSafeUniqueObjectFileName(const char* sin)
+std::string&
+cmLocalGenerator
+::CreateSafeUniqueObjectFileName(const char* sin,
+                                 std::string::size_type dir_len)
 {
   // Look for an existing mapped name for this object file.
   std::map<cmStdString,cmStdString>::iterator it =
@@ -2435,6 +2513,12 @@ std::string& cmLocalGenerator::CreateSafeUniqueObjectFileName(const char* sin)
       while ( !done );
       }
 
+#if defined(CM_LG_ENCODE_OBJECT_NAMES)
+    cmLocalGeneratorCheckObjectName(ssin, dir_len);
+#else
+    (void)dir_len;
+#endif
+
     // Insert the newly mapped object file name.
     std::map<cmStdString, cmStdString>::value_type e(sin, ssin);
     it = this->UniqueObjectNamesMap.insert(e).first;
@@ -2446,7 +2530,9 @@ std::string& cmLocalGenerator::CreateSafeUniqueObjectFileName(const char* sin)
 
 //----------------------------------------------------------------------------
 std::string
-cmLocalGenerator::GetObjectFileNameWithoutTarget(const cmSourceFile& source)
+cmLocalGenerator
+::GetObjectFileNameWithoutTarget(const cmSourceFile& source,
+                                 std::string::size_type dir_len)
 {
   // Construct the object file name using the full path to the source
   // file which is its only unique identification.
@@ -2517,7 +2603,7 @@ cmLocalGenerator::GetObjectFileNameWithoutTarget(const cmSourceFile& source)
     }
 
   // Convert to a safe name.
-  return this->CreateSafeUniqueObjectFileName(objectName.c_str());
+  return this->CreateSafeUniqueObjectFileName(objectName.c_str(), dir_len);
 }
 
 //----------------------------------------------------------------------------

Source/cmLocalGenerator.h

@@ -266,8 +266,10 @@ protected:
     std::vector<std::string> const& configurationTypes);
 
   // Compute object file names.
-  std::string GetObjectFileNameWithoutTarget(const cmSourceFile& source);
-  std::string& CreateSafeUniqueObjectFileName(const char* sin);
+  std::string GetObjectFileNameWithoutTarget(const cmSourceFile& source,
+                                             std::string::size_type dir_len);
+  std::string& CreateSafeUniqueObjectFileName(const char* sin,
+                                              std::string::size_type dir_len);
 
   void ConfigureRelativePaths();
   std::string FindRelativePathTopSource();

Source/cmLocalUnixMakefileGenerator3.cxx

@@ -1610,25 +1610,22 @@ cmLocalUnixMakefileGenerator3
                     const cmSourceFile& source,
                     std::string* nameWithoutTargetDir)
 {
-  // Get the object file name independent of target.
-  std::string objectName = this->GetObjectFileNameWithoutTarget(source);
-  if(nameWithoutTargetDir)
-    {
-    *nameWithoutTargetDir = objectName;
-    }
-
-  // Prepend the target directory.
-  std::string obj;
-  const char* fileTargetDirectory = 
-    source.GetProperty("MACOSX_PACKAGE_LOCATION");
-  if ( fileTargetDirectory )
+  if(const char* fileTargetDirectory =
+     source.GetProperty("MACOSX_PACKAGE_LOCATION"))
     {
+    // Special handling for OSX package files.
+    std::string objectName = this->GetObjectFileNameWithoutTarget(source, 0);
+    if(nameWithoutTargetDir)
+      {
+      *nameWithoutTargetDir = objectName;
+      }
     objectName = cmSystemTools::GetFilenameName(objectName.c_str());
     std::string targetName;
     std::string targetNameReal;
     std::string targetNamePDB;
     target.GetExecutableNames(targetName, targetNameReal,
                               targetNamePDB, this->ConfigurationName.c_str());
+    std::string obj;
     if ( target.GetPropertyAsBool("MACOSX_BUNDLE") )
       {
       // Construct the full path version of the names.
@@ -1644,14 +1641,32 @@ cmLocalUnixMakefileGenerator3
       }
     obj = cmSystemTools::RelativePath
       (this->Makefile->GetHomeOutputDirectory(), obj.c_str());
+    obj += "/";
+    obj += objectName;
+    return obj;
     }
   else
     {
-    obj = this->GetTargetDirectory(target);
+    // Start with the target directory.
+    std::string obj = this->GetTargetDirectory(target);
+    obj += "/";
+
+    // Get the object file name without the target directory.
+    std::string::size_type dir_len = 0;
+    dir_len += strlen(this->Makefile->GetCurrentOutputDirectory());
+    dir_len += 1;
+    dir_len += obj.size();
+    std::string objectName =
+      this->GetObjectFileNameWithoutTarget(source, dir_len);
+    if(nameWithoutTargetDir)
+      {
+      *nameWithoutTargetDir = objectName;
+      }
+
+    // Append the object name to the target directory.
+    obj += objectName;
+    return obj;
     }
-  obj += "/";
-  obj += objectName;
-  return obj;
 }
 
 //----------------------------------------------------------------------------

Source/cmLocalVisualStudio6Generator.cxx

@@ -397,7 +397,32 @@ void cmLocalVisualStudio6Generator
     {
     this->WriteDSPBeginGroup(fout, name.c_str(), "");
     }
-    
+
+  // Compute the maximum length of a configuration name.
+  std::string::size_type config_len_max = 0;
+  for(std::vector<std::string>::iterator i = this->Configurations.begin();
+      i != this->Configurations.end(); ++i)
+    {
+    // Strip the subdirectory name out of the configuration name.
+    std::string config = *i;
+    std::string::size_type pos = config.find_last_of(" ");
+    config = config.substr(pos+1, std::string::npos);
+    config = config.substr(0, config.size()-1);
+    if(config.size() > config_len_max)
+      {
+      config_len_max = config.size();
+      }
+    }
+
+  // Compute the maximum length of the full path to the intermediate
+  // files directory for any configuration.  This is used to construct
+  // object file names that do not produce paths that are too long.
+  std::string::size_type dir_len = 0;
+  dir_len += strlen(this->Makefile->GetCurrentOutputDirectory());
+  dir_len += 1;
+  dir_len += config_len_max;
+  dir_len += 1;
+
   // Loop through each source in the source group.
   for(std::vector<const cmSourceFile *>::const_iterator sf =
         sourceFiles.begin(); sf != sourceFiles.end(); ++sf)
@@ -412,7 +437,7 @@ void cmLocalVisualStudio6Generator
       {
       objectNameDir =
         cmSystemTools::GetFilenamePath(
-          this->GetObjectFileNameWithoutTarget(*(*sf)));
+          this->GetObjectFileNameWithoutTarget(*(*sf), dir_len));
       }
 
     // Add per-source file flags.

Source/cmLocalVisualStudio7Generator.cxx

@@ -1055,6 +1055,28 @@ void cmLocalVisualStudio7Generator
     this->WriteVCProjBeginGroup(fout, name.c_str(), "");
     }
 
+  // Compute the maximum length of a configuration name.
+  std::string::size_type config_len_max = 0;
+  for(std::vector<std::string>::iterator i = configs->begin();
+      i != configs->end(); ++i)
+    {
+    if(i->size() > config_len_max)
+      {
+      config_len_max = i->size();
+      }
+    }
+
+  // Compute the maximum length of the full path to the intermediate
+  // files directory for any configuration.  This is used to construct
+  // object file names that do not produce paths that are too long.
+  std::string::size_type dir_len = 0;
+  dir_len += strlen(this->Makefile->GetCurrentOutputDirectory());
+  dir_len += 1;
+  dir_len += this->GetTargetDirectory(target).size();
+  dir_len += 1;
+  dir_len += config_len_max;
+  dir_len += 1;
+
   // Loop through each source in the source group.
   std::string objectName;
   for(std::vector<const cmSourceFile *>::const_iterator sf =
@@ -1066,7 +1088,7 @@ void cmLocalVisualStudio7Generator
     std::string additionalDeps;
     if(this->NeedObjectName.find(*sf) != this->NeedObjectName.end())
       {
-      objectName = this->GetObjectFileNameWithoutTarget(*(*sf));
+      objectName = this->GetObjectFileNameWithoutTarget(*(*sf), dir_len);
       }
     else
       {

Tests/OutOfSource/OutOfSourceSubdir/CMakeLists.txt

@@ -2,8 +2,26 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
 
 IF ("${PROJECT_SOURCE_DIR}" STREQUAL "${ANOTHER_PROJ_SOURCE_DIR}")
   SET(BUILD_SHARED_LIBS 1)
+
+  # Construct a source file outside the tree whose full path is close to
+  # the path length limit.  This will cause the full path to the object
+  # file in the build tree to exceed the maximum path length which will
+  # test cmLocalGenerator::CreateSafeUniqueObjectFileName.
+  GET_FILENAME_COMPONENT(DEEPDIR
+    ${OutOfSource_BINARY_DIR}/../OutOfSourceDeep/deeper ABSOLUTE)
+  # MAXPATH = 250 less 25 for /and/deeper/simple.cxx part and small safety
+  MATH(EXPR MAXPATH "250 - 25")
+  STRING(LENGTH "${DEEPDIR}" DEEPDIR_LEN)
+  WHILE("${DEEPDIR_LEN}" LESS "${MAXPATH}")
+    SET(DEEPDIR ${DEEPDIR}/and/deeper)
+    STRING(LENGTH "${DEEPDIR}" DEEPDIR_LEN)
+  ENDWHILE("${DEEPDIR_LEN}" LESS "${MAXPATH}")
+  SET(DEEPSRC ${DEEPDIR}/simple.cxx)
+  STRING(LENGTH "${DEEPSRC}" DEEPSRC_LEN)
+  CONFIGURE_FILE(simple.cxx.in ${DEEPSRC} COPYONLY)
+
   ADD_LIBRARY(testlib testlib.cxx)
-  ADD_EXECUTABLE (simple simple.cxx ../simple.cxx)
+  ADD_EXECUTABLE (simple simple.cxx ../simple.cxx ${DEEPSRC})
   TARGET_LINK_LIBRARIES(simple testlib outlib)
 ENDIF ("${PROJECT_SOURCE_DIR}" STREQUAL "${ANOTHER_PROJ_SOURCE_DIR}")
 

Tests/OutOfSource/OutOfSourceSubdir/simple.cxx

@@ -5,6 +5,7 @@
 #include "testdp.h"
 
 extern int simple();
+extern int simple2();
 extern "C" int outlib();
 
 int main ()
@@ -26,5 +27,9 @@ int main ()
     {
     return -4;
     }
+  if(simple2() != 789)
+    {
+    return -5;
+    }
   return 0;
 }

Tests/OutOfSource/OutOfSourceSubdir/simple.cxx.in

@@ -0,0 +1 @@
+int simple2() { return 789; }