Ninja: job pool support for compiling and linking

Could be tested by setting the environment
variable NINJA_STATUS=[%r]

Help/manual/cmake-properties.7.rst

@@ -27,6 +27,7 @@ Properties of Global Scope
    /prop_gbl/IN_TRY_COMPILE
    /prop_gbl/PACKAGES_FOUND
    /prop_gbl/PACKAGES_NOT_FOUND
+   /prop_gbl/JOB_POOLS
    /prop_gbl/PREDEFINED_TARGETS_FOLDER
    /prop_gbl/ECLIPSE_EXTRA_NATURES
    /prop_gbl/REPORT_UNDEFINED_PROPERTIES
@@ -147,6 +148,8 @@ Properties on Targets
    /prop_tgt/INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
    /prop_tgt/INTERPROCEDURAL_OPTIMIZATION_CONFIG
    /prop_tgt/INTERPROCEDURAL_OPTIMIZATION
+   /prop_tgt/JOB_POOL_COMPILE
+   /prop_tgt/JOB_POOL_LINK
    /prop_tgt/LABELS
    /prop_tgt/LANG_VISIBILITY_PRESET
    /prop_tgt/LIBRARY_OUTPUT_DIRECTORY_CONFIG

Help/manual/cmake-variables.7.rst

@@ -41,6 +41,8 @@ Variables that Provide Information
    /variable/CMAKE_HOME_DIRECTORY
    /variable/CMAKE_IMPORT_LIBRARY_PREFIX
    /variable/CMAKE_IMPORT_LIBRARY_SUFFIX
+   /variable/CMAKE_JOB_POOL_COMPILE
+   /variable/CMAKE_JOB_POOL_LINK
    /variable/CMAKE_LINK_LIBRARY_SUFFIX
    /variable/CMAKE_MAJOR_VERSION
    /variable/CMAKE_MAKE_PROGRAM

Help/prop_gbl/JOB_POOLS.rst

@@ -0,0 +1,20 @@
+JOB_POOLS
+---------
+
+Ninja only: List of available pools.
+
+A pool is a named integer property and defines the maximum number
+of concurrent jobs which can be started by a rule assigned to the pool.
+The :prop_gbl:`JOB_POOLS` property is a semicolon-separated list of
+pairs using the syntax NAME=integer (without a space after the equality sign).
+
+For instance:
+
+.. code-block:: cmake
+
+  set_property(GLOBAL PROPERTY JOB_POOLS two_jobs=2 ten_jobs=10)
+
+Defined pools could be used globally by setting
+:variable:`CMAKE_JOB_POOL_COMPILE` and :variable:`CMAKE_JOB_POOL_LINK`
+or per target by setting the target properties
+:prop_tgt:`JOB_POOL_COMPILE` and :prop_tgt:`JOB_POOL_LINK`.

Help/prop_tgt/JOB_POOL_COMPILE.rst

@@ -0,0 +1,17 @@
+JOB_POOL_COMPILE
+----------------
+
+Ninja only: Pool used for compiling.
+
+The number of parallel compile processes could be limited by defining
+pools with the global :prop_gbl:`JOB_POOLS`
+property and then specifying here the pool name.
+
+For instance:
+
+.. code-block:: cmake
+
+  set_property(TARGET myexe PROPERTY JOB_POOL_COMPILE ten_jobs)
+
+This property is initialized by the value of
+:variable:`CMAKE_JOB_POOL_COMPILE`.

Help/prop_tgt/JOB_POOL_LINK.rst

@@ -0,0 +1,16 @@
+JOB_POOL_LINK
+-------------
+
+Ninja only: Pool used for linking.
+
+The number of parallel link processes could be limited by defining
+pools with the global :prop_gbl:`JOB_POOLS`
+property and then specifing here the pool name.
+
+For instance:
+
+.. code-block:: cmake
+
+  set_property(TARGET myexe PROPERTY JOB_POOL_LINK two_jobs)
+
+This property is initialized by the value of :variable:`CMAKE_JOB_POOL_LINK`.

Help/variable/CMAKE_JOB_POOL_COMPILE.rst

@@ -0,0 +1,6 @@
+CMAKE_JOB_POOL_COMPILE
+----------------------
+
+This variable is used to initialize the :prop_tgt:`JOB_POOL_COMPILE`
+property on all the targets. See :prop_tgt:`JOB_POOL_COMPILE`
+for additional information.

Help/variable/CMAKE_JOB_POOL_LINK.rst

@@ -0,0 +1,6 @@
+CMAKE_JOB_POOL_LINK
+----------------------
+
+This variable is used to initialize the :prop_tgt:`JOB_POOL_LINK`
+property on all the targets. See :prop_tgt:`JOB_POOL_LINK`
+for additional information.

Source/cmLocalNinjaGenerator.cxx

@@ -53,6 +53,8 @@ void cmLocalNinjaGenerator::Generate()
     {
     this->WriteBuildFileTop();
 
+    this->WritePools(this->GetRulesFileStream());
+
     const std::string showIncludesPrefix = this->GetMakefile()
              ->GetSafeDefinition("CMAKE_CL_SHOWINCLUDES_PREFIX");
     if (!showIncludesPrefix.empty())
@@ -200,6 +202,39 @@ void cmLocalNinjaGenerator::WriteProjectHeader(std::ostream& os)
   cmGlobalNinjaGenerator::WriteDivider(os);
 }
 
+void cmLocalNinjaGenerator::WritePools(std::ostream& os)
+{
+  cmGlobalNinjaGenerator::WriteDivider(os);
+
+  const char* jobpools = this->GetCMakeInstance()
+                               ->GetProperty("JOB_POOLS", cmProperty::GLOBAL);
+  if (jobpools)
+    {
+    cmGlobalNinjaGenerator::WriteComment(os,
+                            "Pools defined by global property JOB_POOLS");
+    std::vector<std::string> pools;
+    cmSystemTools::ExpandListArgument(jobpools, pools);
+    for (size_t i = 0; i < pools.size(); ++i)
+      {
+      const std::string pool = pools[i];
+      const std::string::size_type eq = pool.find("=");
+      unsigned int jobs;
+      if (eq != std::string::npos &&
+          sscanf(pool.c_str() + eq, "=%u", &jobs) == 1)
+        {
+        os << "pool " << pool.substr(0, eq) << std::endl;
+        os << "  depth = " << jobs << std::endl;
+        os << std::endl;
+        }
+      else
+        {
+        cmSystemTools::Error("Invalid pool defined by property 'JOB_POOLS': ",
+                             pool.c_str());
+        }
+      }
+    }
+}
+
 void cmLocalNinjaGenerator::WriteNinjaFilesInclusion(std::ostream& os)
 {
   cmGlobalNinjaGenerator::WriteDivider(os);

Source/cmLocalNinjaGenerator.h

@@ -112,6 +112,7 @@ private:
   void WriteProjectHeader(std::ostream& os);
   void WriteNinjaFilesInclusion(std::ostream& os);
   void WriteProcessedMakefile(std::ostream& os);
+  void WritePools(std::ostream& os);
 
   void SetConfigName();
 

Source/cmNinjaNormalTargetGenerator.cxx

@@ -464,6 +464,8 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement()
                                             linkPath,
                                             this->GetGeneratorTarget());
 
+  this->addPoolNinjaVariable("JOB_POOL_LINK", this->GetTarget(), vars);
+
   this->AddModuleDefinitionFlag(vars["LINK_FLAGS"]);
   vars["LINK_FLAGS"] = cmGlobalNinjaGenerator
                         ::EncodeLiteral(vars["LINK_FLAGS"]);

Source/cmNinjaTargetGenerator.cxx

@@ -572,6 +572,8 @@ cmNinjaTargetGenerator
                          ConvertToNinjaPath(objectDir.c_str()).c_str(),
                          cmLocalGenerator::SHELL);
 
+  this->addPoolNinjaVariable("JOB_POOL_COMPILE", this->GetTarget(), vars);
+
   this->SetMsvcTargetPdbVariable(vars);
 
   if(this->Makefile->IsOn("CMAKE_EXPORT_COMPILE_COMMANDS"))
@@ -725,3 +727,14 @@ cmNinjaTargetGenerator::MacOSXContentGeneratorType::operator()(
   // Add as a dependency of all target so that it gets called.
   this->Generator->GetGlobalGenerator()->AddDependencyToAll(output);
 }
+
+void cmNinjaTargetGenerator::addPoolNinjaVariable(const char* pool_property,
+                                                  cmTarget* target,
+                                                  cmNinjaVars& vars)
+{
+    const char* pool = target->GetProperty(pool_property);
+    if (pool)
+      {
+      vars["pool"] = pool;
+      }
+}

Source/cmNinjaTargetGenerator.h

@@ -136,12 +136,15 @@ protected:
   };
   friend struct MacOSXContentGeneratorType;
 
-protected:
+
   MacOSXContentGeneratorType* MacOSXContentGenerator;
   // Properly initialized by sub-classes.
   cmOSXBundleGenerator* OSXBundleGenerator;
   std::set<cmStdString> MacContentFolders;
 
+  void addPoolNinjaVariable(const char* pool_property,
+                            cmTarget* target,
+                            cmNinjaVars& vars);
 
 private:
   cmTarget* Target;

Source/cmTarget.cxx

@@ -379,6 +379,9 @@ void cmTarget::SetMakefile(cmMakefile* mf)
     // so ensure that the conditions don't lead to nonsense.
     this->PolicyStatusCMP0022 = cmPolicies::NEW;
     }
+
+  this->SetPropertyDefault("JOB_POOL_COMPILE", 0);
+  this->SetPropertyDefault("JOB_POOL_LINK", 0);
 }
 
 //----------------------------------------------------------------------------