From f0a23aa3dbbe2751bbb4969e1371380561b566ee Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 21 Sep 2016 13:52:01 -0400 Subject: [PATCH 01/10] Ninja: Refactor Fortran rejection logic Delay rejection of Fortran until after we've determined the version of the `ninja` tool to be used. This will later allow us to enable Fortran support based on the version of ninja. While at it, make the rejection an immediate fatal error. Also provide a stack trace so readers know what code tried to enable Fortran. --- Source/cmGlobalGenerator.cxx | 10 ++++++++++ Source/cmGlobalGenerator.h | 2 ++ Source/cmGlobalNinjaGenerator.cxx | 16 +++++++++++++--- Source/cmGlobalNinjaGenerator.h | 2 ++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index 4772474b0..95747f2f7 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -319,6 +319,12 @@ void cmGlobalGenerator::FindMakeProgram(cmMakefile* mf) } } +bool cmGlobalGenerator::CheckLanguages( + std::vector const& /* languages */, cmMakefile* /* mf */) const +{ + return true; +} + // enable the given language // // The following files are loaded in this order: @@ -428,6 +434,10 @@ void cmGlobalGenerator::EnableLanguage( // find and make sure CMAKE_MAKE_PROGRAM is defined this->FindMakeProgram(mf); + if (!this->CheckLanguages(languages, mf)) { + return; + } + // try and load the CMakeSystem.cmake if it is there std::string fpath = rootBin; bool const readCMakeSystem = !mf->GetDefinition("CMAKE_SYSTEM_LOADED"); diff --git a/Source/cmGlobalGenerator.h b/Source/cmGlobalGenerator.h index f7b2e59df..7bc389d87 100644 --- a/Source/cmGlobalGenerator.h +++ b/Source/cmGlobalGenerator.h @@ -381,6 +381,8 @@ protected: void SetLanguageEnabledFlag(const std::string& l, cmMakefile* mf); void SetLanguageEnabledMaps(const std::string& l, cmMakefile* mf); void FillExtensionToLanguageMap(const std::string& l, cmMakefile* mf); + virtual bool CheckLanguages(std::vector const& languages, + cmMakefile* mf) const; virtual void PrintCompilerAdvice(std::ostream& os, std::string const& lang, const char* envVar) const; diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index b9136219f..8146ea568 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -561,12 +561,22 @@ void cmGlobalNinjaGenerator::FindMakeProgram(cmMakefile* mf) } } +bool cmGlobalNinjaGenerator::CheckLanguages( + std::vector const& languages, cmMakefile* mf) const +{ + if (std::find(languages.begin(), languages.end(), "Fortran") != + languages.end()) { + mf->IssueMessage(cmake::FATAL_ERROR, + "The Ninja generator does not support Fortran yet."); + cmSystemTools::SetFatalErrorOccured(); + return false; + } + return true; +} + void cmGlobalNinjaGenerator::EnableLanguage( std::vector const& langs, cmMakefile* mf, bool optional) { - if (std::find(langs.begin(), langs.end(), "Fortran") != langs.end()) { - cmSystemTools::Error("The Ninja generator does not support Fortran yet."); - } this->cmGlobalGenerator::EnableLanguage(langs, mf, optional); for (std::vector::const_iterator l = langs.begin(); l != langs.end(); ++l) { diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h index 02016859d..ac7a6b16d 100644 --- a/Source/cmGlobalNinjaGenerator.h +++ b/Source/cmGlobalNinjaGenerator.h @@ -355,6 +355,8 @@ protected: private: std::string GetEditCacheCommand() const CM_OVERRIDE; void FindMakeProgram(cmMakefile* mf) CM_OVERRIDE; + bool CheckLanguages(std::vector const& languages, + cmMakefile* mf) const CM_OVERRIDE; void OpenBuildFileStream(); void CloseBuildFileStream(); From 0488ae63ea8dfa8ac455e36b7cc41e81620a080f Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 21 Sep 2016 14:22:41 -0400 Subject: [PATCH 02/10] Ninja: Refactor ninja feature detection Check for features as soon as we know the ninja version. Save the results so we do not have to re-compare versions every time. --- Source/cmGlobalNinjaGenerator.cxx | 21 +++++++++++++++------ Source/cmGlobalNinjaGenerator.h | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 8146ea568..77693fefd 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -473,6 +473,8 @@ cmGlobalNinjaGenerator::cmGlobalNinjaGenerator(cmake* cm) , UsingGCCOnWindows(false) , ComputingUnknownDependencies(false) , PolicyCMP0058(cmPolicies::WARN) + , NinjaSupportsConsolePool(false) + , NinjaSupportsImplicitOuts(false) { #ifdef _WIN32 cm->GetState()->SetWindowsShell(true); @@ -558,9 +560,20 @@ void cmGlobalNinjaGenerator::FindMakeProgram(cmMakefile* mf) cmSystemTools::RunSingleCommand(command, &version, CM_NULLPTR, CM_NULLPTR, CM_NULLPTR, cmSystemTools::OUTPUT_NONE); this->NinjaVersion = cmSystemTools::TrimWhitespace(version); + this->CheckNinjaFeatures(); } } +void cmGlobalNinjaGenerator::CheckNinjaFeatures() +{ + this->NinjaSupportsConsolePool = !cmSystemTools::VersionCompare( + cmSystemTools::OP_LESS, this->NinjaVersion.c_str(), + RequiredNinjaVersionForConsolePool().c_str()); + this->NinjaSupportsImplicitOuts = !cmSystemTools::VersionCompare( + cmSystemTools::OP_LESS, this->NinjaVersion.c_str(), + this->RequiredNinjaVersionForImplicitOuts().c_str()); +} + bool cmGlobalNinjaGenerator::CheckLanguages( std::vector const& languages, cmMakefile* mf) const { @@ -1311,16 +1324,12 @@ std::string cmGlobalNinjaGenerator::ninjaCmd() const bool cmGlobalNinjaGenerator::SupportsConsolePool() const { - return !cmSystemTools::VersionCompare( - cmSystemTools::OP_LESS, this->NinjaVersion.c_str(), - RequiredNinjaVersionForConsolePool().c_str()); + return this->NinjaSupportsConsolePool; } bool cmGlobalNinjaGenerator::SupportsImplicitOuts() const { - return !cmSystemTools::VersionCompare( - cmSystemTools::OP_LESS, this->NinjaVersion.c_str(), - this->RequiredNinjaVersionForImplicitOuts().c_str()); + return this->NinjaSupportsImplicitOuts; } void cmGlobalNinjaGenerator::WriteTargetClean(std::ostream& os) diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h index ac7a6b16d..5c7d8fa59 100644 --- a/Source/cmGlobalNinjaGenerator.h +++ b/Source/cmGlobalNinjaGenerator.h @@ -355,6 +355,7 @@ protected: private: std::string GetEditCacheCommand() const CM_OVERRIDE; void FindMakeProgram(cmMakefile* mf) CM_OVERRIDE; + void CheckNinjaFeatures(); bool CheckLanguages(std::vector const& languages, cmMakefile* mf) const CM_OVERRIDE; @@ -441,6 +442,8 @@ private: std::string NinjaCommand; std::string NinjaVersion; + bool NinjaSupportsConsolePool; + bool NinjaSupportsImplicitOuts; private: void InitOutputPathPrefix(); From a57d1bb7122be79101cc17a4b8a04bd0ac5a041e Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 21 Sep 2016 14:38:49 -0400 Subject: [PATCH 03/10] Ninja: Add API to check for dyndep support Kitware maintains a branch of Ninja with support for dynamically discovered dependencies (dyndep) that has not yet been accepted upstream. Add an internal API to check whether the Ninja version in use for the build supports this feature. --- Source/cmGlobalNinjaGenerator.cxx | 11 +++++++++++ Source/cmGlobalNinjaGenerator.h | 1 + 2 files changed, 12 insertions(+) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 77693fefd..42ce6227a 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -475,6 +475,7 @@ cmGlobalNinjaGenerator::cmGlobalNinjaGenerator(cmake* cm) , PolicyCMP0058(cmPolicies::WARN) , NinjaSupportsConsolePool(false) , NinjaSupportsImplicitOuts(false) + , NinjaSupportsDyndeps(0) { #ifdef _WIN32 cm->GetState()->SetWindowsShell(true); @@ -572,6 +573,16 @@ void cmGlobalNinjaGenerator::CheckNinjaFeatures() this->NinjaSupportsImplicitOuts = !cmSystemTools::VersionCompare( cmSystemTools::OP_LESS, this->NinjaVersion.c_str(), this->RequiredNinjaVersionForImplicitOuts().c_str()); + { + // Our ninja branch adds ".dyndep-#" to its version number, + // where '#' is a feature-specific version number. Extract it. + static std::string const k_DYNDEP_ = ".dyndep-"; + std::string::size_type pos = this->NinjaVersion.find(k_DYNDEP_); + if (pos != std::string::npos) { + const char* fv = this->NinjaVersion.c_str() + pos + k_DYNDEP_.size(); + cmSystemTools::StringToULong(fv, &this->NinjaSupportsDyndeps); + } + } } bool cmGlobalNinjaGenerator::CheckLanguages( diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h index 5c7d8fa59..0183952fe 100644 --- a/Source/cmGlobalNinjaGenerator.h +++ b/Source/cmGlobalNinjaGenerator.h @@ -444,6 +444,7 @@ private: std::string NinjaVersion; bool NinjaSupportsConsolePool; bool NinjaSupportsImplicitOuts; + unsigned long NinjaSupportsDyndeps; private: void InitOutputPathPrefix(); From 8eca59a175b551c2d4c98fd9523e6aca6c51a82b Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 22 Sep 2016 10:11:36 -0400 Subject: [PATCH 04/10] Ninja: Add comment with Fortran dependency design documentation --- Source/cmGlobalNinjaGenerator.cxx | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 42ce6227a..6da6617eb 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -1408,3 +1408,81 @@ void cmGlobalNinjaGenerator::StripNinjaOutputPathPrefixAsSuffix( EnsureTrailingSlash(path); cmStripSuffixIfExists(path, this->OutputPathPrefix); } + +/* + +We use the following approach to support Fortran. Each target already +has a .dir/ directory used to hold intermediate files for CMake. +For each target, a FortranDependInfo.json file is generated by CMake with +information about include directories, module directories, and the locations +the per-target directories for target dependencies. + +Compilation of source files within a target is split into the following steps: + +1. Preprocess all sources, scan preprocessed output for module dependencies. + This step is done with independent build statements for each source, + and can therefore be done in parallel. + + rule Fortran_PREPROCESS + depfile = $DEP_FILE + command = gfortran -cpp $DEFINES $INCLUDES $FLAGS -E $in -o $out && + cmake -E cmake_ninja_depends \ + --tdi=FortranDependInfo.json --pp=$out --dep=$DEP_FILE \ + --obj=$OBJ_FILE --ddi=$DYNDEP_INTERMEDIATE_FILE + + build src.f90-pp.f90 | src.f90-pp.f90.ddi: Fortran_PREPROCESS src.f90 + OBJ_FILE = src.f90.o + DEP_FILE = src.f90-pp.f90.d + DYNDEP_INTERMEDIATE_FILE = src.f90-pp.f90.ddi + + The ``cmake -E cmake_ninja_depends`` tool reads the preprocessed output + and generates the ninja depfile for preprocessor dependencies. It also + generates a "ddi" file (in a format private to CMake) that lists the + object file that compilation will produce along with the module names + it provides and/or requires. The "ddi" file is an implicit output + because it should not appear in "$out" but is generated by the rule. + +2. Consolidate the per-source module dependencies saved in the "ddi" + files from all sources to produce a ninja "dyndep" file, ``Fortran.dd``. + + rule Fortran_DYNDEP + command = cmake -E cmake_ninja_dyndep \ + --tdi=FortranDependInfo.json --dd=$out $in + + build Fortran.dd: Fortran_DYNDEP src1.f90-pp.f90.ddi src2.f90-pp.f90.ddi + + The ``cmake -E cmake_ninja_dyndep`` tool reads the "ddi" files from all + sources in the target and the ``FortranModules.json`` files from targets + on which the target depends. It computes dependency edges on compilations + that require modules to those that provide the modules. This information + is placed in the ``Fortran.dd`` file for ninja to load later. It also + writes the expected location of modules provided by this target into + ``FortranModules.json`` for use by dependent targets. + +3. Compile all sources after loading dynamically discovered dependencies + of the compilation build statements from their ``dyndep`` bindings. + + rule Fortran_COMPILE + command = gfortran $INCLUDES $FLAGS -c $in -o $out + + build src1.f90.o: Fortran_COMPILE src1.f90-pp.f90 || Fortran.dd + dyndep = Fortran.dd + + The "dyndep" binding tells ninja to load dynamically discovered + dependency information from ``Fortran.dd``. This adds information + such as: + + build src1.f90.o | mod1.mod: dyndep + restat = 1 + + This tells ninja that ``mod1.mod`` is an implicit output of compiling + the object file ``src1.f90.o``. The ``restat`` binding tells it that + the timestamp of the output may not always change. Additionally: + + build src2.f90.o: dyndep | mod1.mod + + This tells ninja that ``mod1.mod`` is a dependency of compiling the + object file ``src2.f90.o``. This ensures that ``src1.f90.o`` and + ``mod1.mod`` will always be up to date before ``src2.f90.o`` is built + (because the latter consumes the module). +*/ From d3e0b64b1432700a71a10ddf06084f7bb8500694 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 22 Sep 2016 10:14:44 -0400 Subject: [PATCH 05/10] Ninja: Add internal tool to scan Fortran code for module dependencies Create an internal `cmake -E cmake_ninja_depends` tool to scan an already-preprocessed Fortran translation unit for modules that it provides or requires. Save the information in a "ddi" file with a CMake-private format for intermediate dynamic dependency information. This file may the be loaded by another tool to be added later. --- Source/cmGlobalNinjaGenerator.cxx | 125 ++++++++++++++++++++++++++++++ Source/cmcmd.cxx | 10 +++ 2 files changed, 135 insertions(+) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 6da6617eb..e6babe6f4 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -14,6 +14,7 @@ #include "cmAlgorithms.h" #include "cmDocumentationEntry.h" +#include "cmFortranParser.h" #include "cmGeneratedFileStream.h" #include "cmGeneratorExpressionEvaluationFile.h" #include "cmGeneratorTarget.h" @@ -28,6 +29,9 @@ #include "cmVersion.h" #include "cmake.h" +#include "cm_jsoncpp_reader.h" +#include "cm_jsoncpp_writer.h" + #include #include #include @@ -1486,3 +1490,124 @@ Compilation of source files within a target is split into the following steps: ``mod1.mod`` will always be up to date before ``src2.f90.o`` is built (because the latter consumes the module). */ + +int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, + std::vector::const_iterator argEnd) +{ + std::string arg_tdi; + std::string arg_pp; + std::string arg_dep; + std::string arg_obj; + std::string arg_ddi; + for (std::vector::const_iterator a = argBeg; a != argEnd; ++a) { + std::string const& arg = *a; + if (cmHasLiteralPrefix(arg, "--tdi=")) { + arg_tdi = arg.substr(6); + } else if (cmHasLiteralPrefix(arg, "--pp=")) { + arg_pp = arg.substr(5); + } else if (cmHasLiteralPrefix(arg, "--dep=")) { + arg_dep = arg.substr(6); + } else if (cmHasLiteralPrefix(arg, "--obj=")) { + arg_obj = arg.substr(6); + } else if (cmHasLiteralPrefix(arg, "--ddi=")) { + arg_ddi = arg.substr(6); + } else { + cmSystemTools::Error("-E cmake_ninja_depends unknown argument: ", + arg.c_str()); + return 1; + } + } + if (arg_tdi.empty()) { + cmSystemTools::Error("-E cmake_ninja_depends requires value for --tdi="); + return 1; + } + if (arg_pp.empty()) { + cmSystemTools::Error("-E cmake_ninja_depends requires value for --pp="); + return 1; + } + if (arg_dep.empty()) { + cmSystemTools::Error("-E cmake_ninja_depends requires value for --dep="); + return 1; + } + if (arg_obj.empty()) { + cmSystemTools::Error("-E cmake_ninja_depends requires value for --obj="); + return 1; + } + if (arg_ddi.empty()) { + cmSystemTools::Error("-E cmake_ninja_depends requires value for --ddi="); + return 1; + } + + std::vector includes; + { + Json::Value tdio; + Json::Value const& tdi = tdio; + { + cmsys::ifstream tdif(arg_tdi.c_str(), std::ios::in | std::ios::binary); + Json::Reader reader; + if (!reader.parse(tdif, tdio, false)) { + cmSystemTools::Error("-E cmake_ninja_depends failed to parse ", + arg_tdi.c_str(), + reader.getFormattedErrorMessages().c_str()); + return 1; + } + } + + Json::Value const& tdi_include_dirs = tdi["include-dirs"]; + if (tdi_include_dirs.isArray()) { + for (Json::Value::const_iterator i = tdi_include_dirs.begin(); + i != tdi_include_dirs.end(); ++i) { + includes.push_back(i->asString()); + } + } + } + + cmFortranSourceInfo info; + std::set defines; + cmFortranParser parser(includes, defines, info); + if (!cmFortranParser_FilePush(&parser, arg_pp.c_str())) { + cmSystemTools::Error("-E cmake_ninja_depends failed to open ", + arg_pp.c_str()); + return 1; + } + if (cmFortran_yyparse(parser.Scanner) != 0) { + // Failed to parse the file. + return 1; + } + + { + cmGeneratedFileStream depfile(arg_dep.c_str()); + depfile << cmSystemTools::ConvertToUnixOutputPath(arg_pp) << ":"; + for (std::set::iterator i = info.Includes.begin(); + i != info.Includes.end(); ++i) { + depfile << " \\\n " << cmSystemTools::ConvertToUnixOutputPath(*i); + } + depfile << "\n"; + } + + Json::Value ddi(Json::objectValue); + ddi["object"] = arg_obj; + + Json::Value& ddi_provides = ddi["provides"] = Json::arrayValue; + for (std::set::iterator i = info.Provides.begin(); + i != info.Provides.end(); ++i) { + ddi_provides.append(*i); + } + Json::Value& ddi_requires = ddi["requires"] = Json::arrayValue; + for (std::set::iterator i = info.Requires.begin(); + i != info.Requires.end(); ++i) { + // Require modules not provided in the same source. + if (!info.Provides.count(*i)) { + ddi_requires.append(*i); + } + } + + cmGeneratedFileStream ddif(arg_ddi.c_str()); + ddif << ddi; + if (!ddif) { + cmSystemTools::Error("-E cmake_ninja_depends failed to write ", + arg_ddi.c_str()); + return 1; + } + return 0; +} diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index 9daed4b44..998a7ad36 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -52,6 +52,9 @@ #include #include +int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, + std::vector::const_iterator argEnd); + void CMakeCommandUsage(const char* program) { std::ostringstream errorStream; @@ -782,6 +785,13 @@ int cmcmd::ExecuteCMakeCommand(std::vector& args) return cmcmd::ExecuteLinkScript(args); } +#ifdef CMAKE_BUILD_WITH_CMAKE + // Internal CMake ninja dependency scanning support. + else if (args[1] == "cmake_ninja_depends") { + return cmcmd_cmake_ninja_depends(args.begin() + 2, args.end()); + } +#endif + // Internal CMake unimplemented feature notification. else if (args[1] == "cmake_unimplemented_variable") { std::cerr << "Feature not implemented for this platform."; From 0f331d7893bee523e61109661d4e51566f41c350 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 22 Sep 2016 10:19:44 -0400 Subject: [PATCH 06/10] Ninja: Add internal tool to produce a ninja dyndep file for Fortran Create an internal `cmake -E cmake_ninja_dyndep` tool to read the "ddi" files generated by `cmake -E cmake_ninja_depends` from all sources in a target and generate a ninja dyndep file that tells Ninja about Fortran module dependencies within the target and on target dependencies. --- Source/cmGlobalNinjaGenerator.cxx | 220 ++++++++++++++++++++++++++++++ Source/cmGlobalNinjaGenerator.h | 9 ++ Source/cmcmd.cxx | 7 + 3 files changed, 236 insertions(+) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index e6babe6f4..c0503ebef 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -27,6 +27,7 @@ #include "cmTarget.h" #include "cmTargetDepend.h" #include "cmVersion.h" +#include "cm_auto_ptr.hxx" #include "cmake.h" #include "cm_jsoncpp_reader.h" @@ -1611,3 +1612,222 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, } return 0; } + +struct cmFortranObjectInfo +{ + std::string Object; + std::vector Provides; + std::vector Requires; +}; + +bool cmGlobalNinjaGenerator::WriteDyndepFile( + std::string const& dir_top_src, std::string const& dir_top_bld, + std::string const& dir_cur_src, std::string const& dir_cur_bld, + std::string const& arg_dd, std::vector const& arg_ddis, + std::string const& module_dir, + std::vector const& linked_target_dirs) +{ + // Setup path conversions. + { + cmState::Snapshot snapshot = + this->GetCMakeInstance()->GetCurrentSnapshot(); + snapshot.GetDirectory().SetCurrentSource(dir_cur_src); + snapshot.GetDirectory().SetCurrentBinary(dir_cur_bld); + snapshot.GetDirectory().SetRelativePathTopSource(dir_top_src.c_str()); + snapshot.GetDirectory().SetRelativePathTopBinary(dir_top_bld.c_str()); + CM_AUTO_PTR mfd(new cmMakefile(this, snapshot)); + CM_AUTO_PTR lgd(static_cast( + this->CreateLocalGenerator(mfd.get()))); + this->Makefiles.push_back(mfd.release()); + this->LocalGenerators.push_back(lgd.release()); + } + + std::vector objects; + for (std::vector::const_iterator ddii = arg_ddis.begin(); + ddii != arg_ddis.end(); ++ddii) { + // Load the ddi file and compute the module file paths it provides. + Json::Value ddio; + Json::Value const& ddi = ddio; + cmsys::ifstream ddif(ddii->c_str(), std::ios::in | std::ios::binary); + Json::Reader reader; + if (!reader.parse(ddif, ddio, false)) { + cmSystemTools::Error("-E cmake_ninja_dyndep failed to parse ", + ddii->c_str(), + reader.getFormattedErrorMessages().c_str()); + return false; + } + + cmFortranObjectInfo info; + info.Object = ddi["object"].asString(); + Json::Value const& ddi_provides = ddi["provides"]; + if (ddi_provides.isArray()) { + for (Json::Value::const_iterator i = ddi_provides.begin(); + i != ddi_provides.end(); ++i) { + info.Provides.push_back(i->asString()); + } + } + Json::Value const& ddi_requires = ddi["requires"]; + if (ddi_requires.isArray()) { + for (Json::Value::const_iterator i = ddi_requires.begin(); + i != ddi_requires.end(); ++i) { + info.Requires.push_back(i->asString()); + } + } + objects.push_back(info); + } + + // Map from module name to module file path, if known. + std::map mod_files; + + // Populate the module map with those provided by linked targets first. + for (std::vector::const_iterator di = + linked_target_dirs.begin(); + di != linked_target_dirs.end(); ++di) { + std::string const ltmn = *di + "/FortranModules.json"; + Json::Value ltm; + cmsys::ifstream ltmf(ltmn.c_str(), std::ios::in | std::ios::binary); + Json::Reader reader; + if (ltmf && !reader.parse(ltmf, ltm, false)) { + cmSystemTools::Error("-E cmake_ninja_dyndep failed to parse ", + di->c_str(), + reader.getFormattedErrorMessages().c_str()); + return false; + } + if (ltm.isObject()) { + for (Json::Value::iterator i = ltm.begin(); i != ltm.end(); ++i) { + mod_files[i.key().asString()] = i->asString(); + } + } + } + + // Extend the module map with those provided by this target. + // We do this after loading the modules provided by linked targets + // in case we have one of the same name that must be preferred. + Json::Value tm = Json::objectValue; + for (std::vector::iterator oi = objects.begin(); + oi != objects.end(); ++oi) { + for (std::vector::iterator i = oi->Provides.begin(); + i != oi->Provides.end(); ++i) { + std::string const mod = module_dir + *i + ".mod"; + mod_files[*i] = mod; + tm[*i] = mod; + } + } + + cmGeneratedFileStream ddf(arg_dd.c_str()); + ddf << "ninja_dyndep_version = 1.0\n"; + + for (std::vector::iterator oi = objects.begin(); + oi != objects.end(); ++oi) { + std::string const ddComment; + std::string const ddRule = "dyndep"; + cmNinjaDeps ddOutputs; + cmNinjaDeps ddImplicitOuts; + cmNinjaDeps ddExplicitDeps; + cmNinjaDeps ddImplicitDeps; + cmNinjaDeps ddOrderOnlyDeps; + cmNinjaVars ddVars; + + ddOutputs.push_back(oi->Object); + for (std::vector::iterator i = oi->Provides.begin(); + i != oi->Provides.end(); ++i) { + ddImplicitOuts.push_back(this->ConvertToNinjaPath(mod_files[*i])); + } + for (std::vector::iterator i = oi->Requires.begin(); + i != oi->Requires.end(); ++i) { + std::map::iterator m = mod_files.find(*i); + if (m != mod_files.end()) { + ddImplicitDeps.push_back(this->ConvertToNinjaPath(m->second)); + } + } + if (!oi->Provides.empty()) { + ddVars["restat"] = "1"; + } + + this->WriteBuild(ddf, ddComment, ddRule, ddOutputs, ddImplicitOuts, + ddExplicitDeps, ddImplicitDeps, ddOrderOnlyDeps, ddVars); + } + + // Store the map of modules provided by this target in a file for + // use by dependents that reference this target in linked-target-dirs. + std::string const target_mods_file = + cmSystemTools::GetFilenamePath(arg_dd) + "/FortranModules.json"; + cmGeneratedFileStream tmf(target_mods_file.c_str()); + tmf << tm; + + return true; +} + +int cmcmd_cmake_ninja_dyndep(std::vector::const_iterator argBeg, + std::vector::const_iterator argEnd) +{ + std::string arg_dd; + std::string arg_tdi; + std::vector arg_ddis; + for (std::vector::const_iterator a = argBeg; a != argEnd; ++a) { + std::string const& arg = *a; + if (cmHasLiteralPrefix(arg, "--tdi=")) { + arg_tdi = arg.substr(6); + } else if (cmHasLiteralPrefix(arg, "--dd=")) { + arg_dd = arg.substr(5); + } else if (!cmHasLiteralPrefix(arg, "--") && + cmHasLiteralSuffix(arg, ".ddi")) { + arg_ddis.push_back(arg); + } else { + cmSystemTools::Error("-E cmake_ninja_dyndep unknown argument: ", + arg.c_str()); + return 1; + } + } + if (arg_tdi.empty()) { + cmSystemTools::Error("-E cmake_ninja_dyndep requires value for --tdi="); + return 1; + } + if (arg_dd.empty()) { + cmSystemTools::Error("-E cmake_ninja_dyndep requires value for --dd="); + return 1; + } + + Json::Value tdio; + Json::Value const& tdi = tdio; + { + cmsys::ifstream tdif(arg_tdi.c_str(), std::ios::in | std::ios::binary); + Json::Reader reader; + if (!reader.parse(tdif, tdio, false)) { + cmSystemTools::Error("-E cmake_ninja_dyndep failed to parse ", + arg_tdi.c_str(), + reader.getFormattedErrorMessages().c_str()); + return 1; + } + } + + std::string const dir_cur_bld = tdi["dir-cur-bld"].asString(); + std::string const dir_cur_src = tdi["dir-cur-src"].asString(); + std::string const dir_top_bld = tdi["dir-top-bld"].asString(); + std::string const dir_top_src = tdi["dir-top-src"].asString(); + std::string module_dir = tdi["module-dir"].asString(); + if (!module_dir.empty()) { + module_dir += "/"; + } + std::vector linked_target_dirs; + Json::Value const& tdi_linked_target_dirs = tdi["linked-target-dirs"]; + if (tdi_linked_target_dirs.isArray()) { + for (Json::Value::const_iterator i = tdi_linked_target_dirs.begin(); + i != tdi_linked_target_dirs.end(); ++i) { + linked_target_dirs.push_back(i->asString()); + } + } + + cmake cm; + cm.SetHomeDirectory(dir_top_src); + cm.SetHomeOutputDirectory(dir_top_bld); + CM_AUTO_PTR ggd( + static_cast(cm.CreateGlobalGenerator("Ninja"))); + if (!ggd.get() || + !ggd->WriteDyndepFile(dir_top_src, dir_top_bld, dir_cur_src, dir_cur_bld, + arg_dd, arg_ddis, module_dir, + linked_target_dirs)) { + return 1; + } + return 0; +} diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h index 0183952fe..6fb93e4c8 100644 --- a/Source/cmGlobalNinjaGenerator.h +++ b/Source/cmGlobalNinjaGenerator.h @@ -347,6 +347,15 @@ public: bool HasOutputPathPrefix() const { return !this->OutputPathPrefix.empty(); } void StripNinjaOutputPathPrefixAsSuffix(std::string& path); + bool WriteDyndepFile(std::string const& dir_top_src, + std::string const& dir_top_bld, + std::string const& dir_cur_src, + std::string const& dir_cur_bld, + std::string const& arg_dd, + std::vector const& arg_ddis, + std::string const& module_dir, + std::vector const& linked_target_dirs); + protected: void Generate() CM_OVERRIDE; diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index 998a7ad36..2b4a83005 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -54,6 +54,8 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, std::vector::const_iterator argEnd); +int cmcmd_cmake_ninja_dyndep(std::vector::const_iterator argBeg, + std::vector::const_iterator argEnd); void CMakeCommandUsage(const char* program) { @@ -790,6 +792,11 @@ int cmcmd::ExecuteCMakeCommand(std::vector& args) else if (args[1] == "cmake_ninja_depends") { return cmcmd_cmake_ninja_depends(args.begin() + 2, args.end()); } + + // Internal CMake ninja dyndep support. + else if (args[1] == "cmake_ninja_dyndep") { + return cmcmd_cmake_ninja_dyndep(args.begin() + 2, args.end()); + } #endif // Internal CMake unimplemented feature notification. From 9a77680eed49939f8ba418af96eefd42ecea0ae1 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 21 Sep 2016 15:05:34 -0400 Subject: [PATCH 07/10] Ninja: Conditionally allow Fortran based on ninja 'dyndep' support Detect from the version of Ninja whether it supports the dynamically discovered dependencies (dyndep) feature needed to support Fortran. --- Source/cmGlobalNinjaGenerator.cxx | 43 ++++++++++++++++++++++++++++--- Source/cmGlobalNinjaGenerator.h | 1 + 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index c0503ebef..81690e7d8 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -595,14 +595,49 @@ bool cmGlobalNinjaGenerator::CheckLanguages( { if (std::find(languages.begin(), languages.end(), "Fortran") != languages.end()) { - mf->IssueMessage(cmake::FATAL_ERROR, - "The Ninja generator does not support Fortran yet."); - cmSystemTools::SetFatalErrorOccured(); - return false; + return this->CheckFortran(mf); } return true; } +bool cmGlobalNinjaGenerator::CheckFortran(cmMakefile* mf) const +{ + if (this->NinjaSupportsDyndeps == 1) { + return true; + } + + std::ostringstream e; + if (this->NinjaSupportsDyndeps == 0) { + /* clang-format off */ + e << + "The Ninja generator does not support Fortran using Ninja version\n" + " " + this->NinjaVersion + "\n" + "due to lack of required features. " + "Kitware has implemented the required features but as of this version " + "of CMake they have not been integrated to upstream ninja. " + "Pending integration, Kitware maintains a branch at:\n" + " https://github.com/Kitware/ninja/tree/features-for-fortran#readme\n" + "with the required features. " + "One may build ninja from that branch to get support for Fortran." + ; + /* clang-format on */ + } else { + /* clang-format off */ + e << + "The Ninja generator in this version of CMake does not support Fortran " + "using Ninja version\n" + " " + this->NinjaVersion + "\n" + "because its 'dyndep' feature version is " << + this->NinjaSupportsDyndeps << ". " + "This version of CMake is aware only of 'dyndep' feature version 1." + ; + /* clang-format on */ + } + mf->IssueMessage(cmake::FATAL_ERROR, e.str()); + cmSystemTools::SetFatalErrorOccured(); + return false; +} + void cmGlobalNinjaGenerator::EnableLanguage( std::vector const& langs, cmMakefile* mf, bool optional) { diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h index 6fb93e4c8..76430a08f 100644 --- a/Source/cmGlobalNinjaGenerator.h +++ b/Source/cmGlobalNinjaGenerator.h @@ -367,6 +367,7 @@ private: void CheckNinjaFeatures(); bool CheckLanguages(std::vector const& languages, cmMakefile* mf) const CM_OVERRIDE; + bool CheckFortran(cmMakefile* mf) const; void OpenBuildFileStream(); void CloseBuildFileStream(); From 39ebfc79e614dc395d5ace2ad5818b3ba75ca478 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 21 Sep 2016 15:38:53 -0400 Subject: [PATCH 08/10] Ninja: Add explicit preprocessing step for Fortran All Fortran sources need to be preprocessed before any source may be compiled so that module dependencies can be (later) extracted. Factor out an explicit preprocessing step preceding compilation. Use Ninja depfile dependencies on the preprocessing step and then compile the already-preprocessed source with a separate build statement that depends explicitly only on the preprocessor output. Later we will insert dynamic discovery of module dependencies between these steps. --- Modules/Compiler/GNU-Fortran.cmake | 3 + Modules/Compiler/Intel-Fortran.cmake | 3 + Modules/Compiler/SunPro-Fortran.cmake | 3 + Source/cmNinjaTargetGenerator.cxx | 166 +++++++++++++++++++++++++- Source/cmNinjaTargetGenerator.h | 5 + 5 files changed, 179 insertions(+), 1 deletion(-) diff --git a/Modules/Compiler/GNU-Fortran.cmake b/Modules/Compiler/GNU-Fortran.cmake index fc848ac42..94dc2755a 100644 --- a/Modules/Compiler/GNU-Fortran.cmake +++ b/Modules/Compiler/GNU-Fortran.cmake @@ -1,6 +1,9 @@ include(Compiler/GNU) __compiler_gnu(Fortran) +set(CMAKE_Fortran_PREPROCESS_SOURCE + " -cpp -E -o ") + set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form") set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form") diff --git a/Modules/Compiler/Intel-Fortran.cmake b/Modules/Compiler/Intel-Fortran.cmake index ef7aa3a6c..a1320555a 100644 --- a/Modules/Compiler/Intel-Fortran.cmake +++ b/Modules/Compiler/Intel-Fortran.cmake @@ -7,3 +7,6 @@ set(CMAKE_Fortran_FORMAT_FREE_FLAG "-free") set(CMAKE_Fortran_CREATE_PREPROCESSED_SOURCE " -E > ") set(CMAKE_Fortran_CREATE_ASSEMBLY_SOURCE " -S -o ") + +set(CMAKE_Fortran_PREPROCESS_SOURCE + " -fpp -E > ") diff --git a/Modules/Compiler/SunPro-Fortran.cmake b/Modules/Compiler/SunPro-Fortran.cmake index a0e07d4cb..6607926a9 100644 --- a/Modules/Compiler/SunPro-Fortran.cmake +++ b/Modules/Compiler/SunPro-Fortran.cmake @@ -18,5 +18,8 @@ string(APPEND CMAKE_Fortran_FLAGS_RELWITHDEBINFO_INIT " -g -xO2 -DNDEBUG") set(CMAKE_Fortran_MODDIR_FLAG "-moddir=") set(CMAKE_Fortran_MODPATH_FLAG "-M") +set(CMAKE_Fortran_PREPROCESS_SOURCE + " -F -o ") + set(CMAKE_Fortran_CREATE_PREPROCESSED_SOURCE " -F -o ") set(CMAKE_Fortran_CREATE_ASSEMBLY_SOURCE " -S -o ") diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx index fb09bfe25..76bae6b55 100644 --- a/Source/cmNinjaTargetGenerator.cxx +++ b/Source/cmNinjaTargetGenerator.cxx @@ -93,6 +93,19 @@ std::string cmNinjaTargetGenerator::LanguageCompilerRule( cmGlobalNinjaGenerator::EncodeRuleName(this->GeneratorTarget->GetName()); } +std::string cmNinjaTargetGenerator::LanguagePreprocessRule( + std::string const& lang) const +{ + return lang + "_PREPROCESS__" + + cmGlobalNinjaGenerator::EncodeRuleName(this->GeneratorTarget->GetName()); +} + +bool cmNinjaTargetGenerator::NeedExplicitPreprocessing( + std::string const& lang) const +{ + return lang == "Fortran"; +} + std::string cmNinjaTargetGenerator::OrderDependsTargetForTarget() { return "cmake_order_depends_target_" + this->GetTargetName(); @@ -229,6 +242,41 @@ std::string cmNinjaTargetGenerator::GetObjectFilePath( return path; } +std::string cmNinjaTargetGenerator::GetPreprocessedFilePath( + cmSourceFile const* source) const +{ + // Choose an extension to compile already-preprocessed source. + std::string ppExt = source->GetExtension(); + if (cmHasLiteralPrefix(ppExt, "F")) { + // Some Fortran compilers automatically enable preprocessing for + // upper-case extensions. Since the source is already preprocessed, + // use a lower-case extension. + ppExt = cmSystemTools::LowerCase(ppExt); + } + if (ppExt == "fpp") { + // Some Fortran compilers automatically enable preprocessing for + // the ".fpp" extension. Since the source is already preprocessed, + // use the ".f" extension. + ppExt = "f"; + } + + // Take the object file name and replace the extension. + std::string const& objName = this->GeneratorTarget->GetObjectName(source); + std::string const& objExt = + this->GetGlobalGenerator()->GetLanguageOutputExtension(*source); + assert(objName.size() >= objExt.size()); + std::string const ppName = + objName.substr(0, objName.size() - objExt.size()) + "-pp." + ppExt; + + std::string path = this->LocalGenerator->GetHomeRelativeOutputPath(); + if (!path.empty()) + path += "/"; + path += this->LocalGenerator->GetTargetDirectory(this->GeneratorTarget); + path += "/"; + path += ppName; + return path; +} + std::string cmNinjaTargetGenerator::GetTargetOutputDir() const { std::string dir = this->GeneratorTarget->GetDirectory(this->GetConfigName()); @@ -311,6 +359,9 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) vars.ObjectDir = "$OBJECT_DIR"; vars.ObjectFileDir = "$OBJECT_FILE_DIR"; + // For some cases we do an explicit preprocessor invocation. + bool const explicitPP = this->NeedExplicitPreprocessing(lang); + cmMakefile* mf = this->GetMakefile(); std::string flags = "$FLAGS"; @@ -331,7 +382,9 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) std::string deptype; std::string depfile; std::string cldeps; - if (this->NeedDepTypeMSVC(lang)) { + if (explicitPP) { + // The explicit preprocessing step will handle dependency scanning. + } else if (this->NeedDepTypeMSVC(lang)) { deptype = "msvc"; depfile = ""; flags += " /showIncludes"; @@ -371,6 +424,66 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) vars.Flags = flags.c_str(); vars.DependencyFile = depfile.c_str(); + if (explicitPP) { + // Lookup the explicit preprocessing rule. + std::string const ppVar = "CMAKE_" + lang + "_PREPROCESS_SOURCE"; + std::string const ppCmd = + this->GetMakefile()->GetRequiredDefinition(ppVar); + + // Explicit preprocessing always uses a depfile. + std::string const ppDeptype = "gcc"; + std::string const ppDepfile = "$DEP_FILE"; + + cmLocalGenerator::RuleVariables ppVars; + ppVars.RuleLauncher = vars.RuleLauncher; + ppVars.CMTarget = vars.CMTarget; + ppVars.Language = vars.Language; + ppVars.Object = "$out"; // for RULE_LAUNCH_COMPILE + ppVars.PreprocessedSource = "$out"; + ppVars.DependencyFile = ppDepfile.c_str(); + + // Preprocessing uses the original source, + // compilation uses preprocessed output. + ppVars.Source = vars.Source; + vars.Source = "$in"; + + // Preprocessing and compilation use the same flags. + ppVars.Flags = vars.Flags; + + // Move preprocessor definitions to the preprocessor rule. + ppVars.Defines = vars.Defines; + vars.Defines = ""; + + // Copy include directories to the preprocessor rule. The Fortran + // compilation rule still needs them for the INCLUDE directive. + ppVars.Includes = vars.Includes; + + // Rule for preprocessing source file. + std::vector ppCmds; + cmSystemTools::ExpandListArgument(ppCmd, ppCmds); + + for (std::vector::iterator i = ppCmds.begin(); + i != ppCmds.end(); ++i) { + this->GetLocalGenerator()->ExpandRuleVariables(*i, ppVars); + } + + std::string const ppCmdLine = + this->GetLocalGenerator()->BuildCommandLine(ppCmds); + + // Write the rule for preprocessing file of the given language. + std::ostringstream ppComment; + ppComment << "Rule for preprocessing " << lang << " files."; + std::ostringstream ppDesc; + ppDesc << "Building " << lang << " preprocessed $out"; + this->GetGlobalGenerator()->AddRule(this->LanguagePreprocessRule(lang), + ppCmdLine, ppDesc.str(), + ppComment.str(), ppDepfile, ppDeptype, + /*rspfile*/ "", + /*rspcontent*/ "", + /*restat*/ "", + /*generator*/ false); + } + // Rule for compiling object file. const std::string cmdVar = std::string("CMAKE_") + lang + "_COMPILE_OBJECT"; std::string compileCmd = mf->GetRequiredDefinition(cmdVar); @@ -589,6 +702,57 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement( orderOnlyDeps); } + // For some cases we do an explicit preprocessor invocation. + bool const explicitPP = this->NeedExplicitPreprocessing(language); + if (explicitPP) { + std::string const ppComment; + std::string const ppRule = this->LanguagePreprocessRule(language); + cmNinjaDeps ppOutputs; + cmNinjaDeps ppImplicitOuts; + cmNinjaDeps ppExplicitDeps; + cmNinjaDeps ppImplicitDeps; + cmNinjaDeps ppOrderOnlyDeps; + cmNinjaVars ppVars; + + std::string const ppFileName = + this->ConvertToNinjaPath(this->GetPreprocessedFilePath(source)); + ppOutputs.push_back(ppFileName); + + // Move compilation dependencies to the preprocessing build statement. + std::swap(ppExplicitDeps, explicitDeps); + std::swap(ppImplicitDeps, implicitDeps); + std::swap(ppOrderOnlyDeps, orderOnlyDeps); + std::swap(ppVars["IN_ABS"], vars["IN_ABS"]); + + // The actual compilation will now use the preprocessed source. + explicitDeps.push_back(ppFileName); + + // Preprocessing and compilation use the same flags. + ppVars["FLAGS"] = vars["FLAGS"]; + + // Move preprocessor definitions to the preprocessor build statement. + std::swap(ppVars["DEFINES"], vars["DEFINES"]); + + // Copy include directories to the preprocessor build statement. The + // Fortran compilation build statement still needs them for the INCLUDE + // directive. + ppVars["INCLUDES"] = vars["INCLUDES"]; + + // Explicit preprocessing always uses a depfile. + ppVars["DEP_FILE"] = + cmGlobalNinjaGenerator::EncodeDepfileSpace(ppFileName + ".d"); + // The actual compilation does not need a depfile because it + // depends on the already-preprocessed source. + vars.erase("DEP_FILE"); + + this->addPoolNinjaVariable("JOB_POOL_COMPILE", this->GetGeneratorTarget(), + ppVars); + + this->GetGlobalGenerator()->WriteBuild( + this->GetBuildFileStream(), ppComment, ppRule, ppOutputs, ppImplicitOuts, + ppExplicitDeps, ppImplicitDeps, ppOrderOnlyDeps, ppVars); + } + EnsureParentDirectoryExists(objectFileName); vars["OBJECT_DIR"] = this->GetLocalGenerator()->ConvertToOutputFormat( diff --git a/Source/cmNinjaTargetGenerator.h b/Source/cmNinjaTargetGenerator.h index 2b2678850..e6816db86 100644 --- a/Source/cmNinjaTargetGenerator.h +++ b/Source/cmNinjaTargetGenerator.h @@ -70,6 +70,8 @@ protected: cmMakefile* GetMakefile() const { return this->Makefile; } std::string LanguageCompilerRule(const std::string& lang) const; + std::string LanguagePreprocessRule(std::string const& lang) const; + bool NeedExplicitPreprocessing(std::string const& lang) const; std::string OrderDependsTargetForTarget(); @@ -107,6 +109,9 @@ protected: /// @return the object file path for the given @a source. std::string GetObjectFilePath(cmSourceFile const* source) const; + /// @return the preprocessed source file path for the given @a source. + std::string GetPreprocessedFilePath(cmSourceFile const* source) const; + /// @return the file path where the target named @a name is generated. std::string GetTargetFilePath(const std::string& name) const; From 59aae29214b727944978b8e545a552f96c88b883 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 22 Sep 2016 10:34:45 -0400 Subject: [PATCH 09/10] Ninja: Add dyndep rules for Fortran module dependencies Teach the Ninja generator to add dyndep rules and bindings as described in the design comment in `Source/cmGlobalNinjaGenerator.cxx`. --- Source/cmNinjaTargetGenerator.cxx | 165 +++++++++++++++++++++++++++++- Source/cmNinjaTargetGenerator.h | 10 ++ 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx index 76bae6b55..b418ce380 100644 --- a/Source/cmNinjaTargetGenerator.cxx +++ b/Source/cmNinjaTargetGenerator.cxx @@ -29,6 +29,8 @@ #include "cmSystemTools.h" #include "cmake.h" +#include "cm_jsoncpp_writer.h" + #include #include #include @@ -106,6 +108,18 @@ bool cmNinjaTargetGenerator::NeedExplicitPreprocessing( return lang == "Fortran"; } +std::string cmNinjaTargetGenerator::LanguageDyndepRule( + const std::string& lang) const +{ + return lang + "_DYNDEP__" + + cmGlobalNinjaGenerator::EncodeRuleName(this->GeneratorTarget->GetName()); +} + +bool cmNinjaTargetGenerator::NeedDyndep(std::string const& lang) const +{ + return lang == "Fortran"; +} + std::string cmNinjaTargetGenerator::OrderDependsTargetForTarget() { return "cmake_order_depends_target_" + this->GetTargetName(); @@ -277,6 +291,29 @@ std::string cmNinjaTargetGenerator::GetPreprocessedFilePath( return path; } +std::string cmNinjaTargetGenerator::GetDyndepFilePath( + std::string const& lang) const +{ + std::string path = this->LocalGenerator->GetHomeRelativeOutputPath(); + if (!path.empty()) + path += "/"; + path += this->LocalGenerator->GetTargetDirectory(this->GeneratorTarget); + path += "/"; + path += lang; + path += ".dd"; + return path; +} + +std::string cmNinjaTargetGenerator::GetTargetDependInfoPath( + std::string const& lang) const +{ + std::string path = this->Makefile->GetCurrentBinaryDirectory(); + path += "/"; + path += this->LocalGenerator->GetTargetDirectory(this->GeneratorTarget); + path += "/" + lang + "DependInfo.json"; + return path; +} + std::string cmNinjaTargetGenerator::GetTargetOutputDir() const { std::string dir = this->GeneratorTarget->GetDirectory(this->GetConfigName()); @@ -361,6 +398,7 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) // For some cases we do an explicit preprocessor invocation. bool const explicitPP = this->NeedExplicitPreprocessing(lang); + bool const needDyndep = this->NeedDyndep(lang); cmMakefile* mf = this->GetMakefile(); @@ -424,6 +462,10 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) vars.Flags = flags.c_str(); vars.DependencyFile = depfile.c_str(); + std::string const tdi = this->GetLocalGenerator()->ConvertToOutputFormat( + ConvertToNinjaPath(this->GetTargetDependInfoPath(lang)), + cmLocalGenerator::SHELL); + if (explicitPP) { // Lookup the explicit preprocessing rule. std::string const ppVar = "CMAKE_" + lang + "_PREPROCESS_SOURCE"; @@ -431,7 +473,7 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) this->GetMakefile()->GetRequiredDefinition(ppVar); // Explicit preprocessing always uses a depfile. - std::string const ppDeptype = "gcc"; + std::string const ppDeptype = ""; // no deps= for multiple outputs std::string const ppDepfile = "$DEP_FILE"; cmLocalGenerator::RuleVariables ppVars; @@ -467,6 +509,16 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) this->GetLocalGenerator()->ExpandRuleVariables(*i, ppVars); } + // Run CMake dependency scanner on preprocessed output. + std::string const cmake = this->GetLocalGenerator()->ConvertToOutputFormat( + cmSystemTools::GetCMakeCommand(), cmLocalGenerator::SHELL); + ppCmds.push_back( + cmake + " -E cmake_ninja_depends" + " --tdi=" + + tdi + " --pp=$out" + " --dep=$DEP_FILE" + + (needDyndep ? " --obj=$OBJ_FILE --ddi=$DYNDEP_INTERMEDIATE_FILE" : "")); + std::string const ppCmdLine = this->GetLocalGenerator()->BuildCommandLine(ppCmds); @@ -484,6 +536,35 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang) /*generator*/ false); } + if (needDyndep) { + // Write the rule for ninja dyndep file generation. + std::vector ddCmds; + + // Run CMake dependency scanner on preprocessed output. + std::string const cmake = this->GetLocalGenerator()->ConvertToOutputFormat( + cmSystemTools::GetCMakeCommand(), cmLocalGenerator::SHELL); + ddCmds.push_back(cmake + " -E cmake_ninja_dyndep" + " --tdi=" + + tdi + " --dd=$out" + " $in"); + + std::string const ddCmdLine = + this->GetLocalGenerator()->BuildCommandLine(ddCmds); + + std::ostringstream ddComment; + ddComment << "Rule to generate ninja dyndep files for " << lang << "."; + std::ostringstream ddDesc; + ddDesc << "Generating " << lang << " dyndep file $out"; + this->GetGlobalGenerator()->AddRule( + this->LanguageDyndepRule(lang), ddCmdLine, ddDesc.str(), ddComment.str(), + /*depfile*/ "", + /*deps*/ "", + /*rspfile*/ "", + /*rspcontent*/ "", + /*restat*/ "", + /*generator*/ false); + } + // Rule for compiling object file. const std::string cmdVar = std::string("CMAKE_") + lang + "_COMPILE_OBJECT"; std::string compileCmd = mf->GetRequiredDefinition(cmdVar); @@ -632,6 +713,25 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatements() this->WriteObjectBuildStatement(*si, !orderOnlyDeps.empty()); } + if (!this->DDIFiles.empty()) { + std::string const ddComment; + std::string const ddRule = this->LanguageDyndepRule("Fortran"); + cmNinjaDeps ddOutputs; + cmNinjaDeps ddImplicitOuts; + cmNinjaDeps const& ddExplicitDeps = this->DDIFiles; + cmNinjaDeps ddImplicitDeps; + cmNinjaDeps ddOrderOnlyDeps; + cmNinjaVars ddVars; + + this->WriteTargetDependInfo("Fortran"); + + ddOutputs.push_back(this->GetDyndepFilePath("Fortran")); + + this->GetGlobalGenerator()->WriteBuild( + this->GetBuildFileStream(), ddComment, ddRule, ddOutputs, ddImplicitOuts, + ddExplicitDeps, ddImplicitDeps, ddOrderOnlyDeps, ddVars); + } + this->GetBuildFileStream() << "\n"; } @@ -702,6 +802,9 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement( orderOnlyDeps); } + // For some cases we need to generate a ninja dyndep file. + bool const needDyndep = this->NeedDyndep(language); + // For some cases we do an explicit preprocessor invocation. bool const explicitPP = this->NeedExplicitPreprocessing(language); if (explicitPP) { @@ -745,6 +848,18 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement( // depends on the already-preprocessed source. vars.erase("DEP_FILE"); + if (needDyndep) { + // Tell dependency scanner the object file that will result from + // compiling the preprocessed source. + ppVars["OBJ_FILE"] = objectFileName; + + // Tell dependency scanner where to store dyndep intermediate results. + std::string const ddiFile = ppFileName + ".ddi"; + ppVars["DYNDEP_INTERMEDIATE_FILE"] = ddiFile; + ppImplicitOuts.push_back(ddiFile); + this->DDIFiles.push_back(ddiFile); + } + this->addPoolNinjaVariable("JOB_POOL_COMPILE", this->GetGeneratorTarget(), ppVars); @@ -752,6 +867,11 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement( this->GetBuildFileStream(), ppComment, ppRule, ppOutputs, ppImplicitOuts, ppExplicitDeps, ppImplicitDeps, ppOrderOnlyDeps, ppVars); } + if (needDyndep) { + std::string const dyndep = this->GetDyndepFilePath(language); + orderOnlyDeps.push_back(dyndep); + vars["dyndep"] = dyndep; + } EnsureParentDirectoryExists(objectFileName); @@ -786,6 +906,49 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement( } } +void cmNinjaTargetGenerator::WriteTargetDependInfo(std::string const& lang) +{ + Json::Value tdi(Json::objectValue); + tdi["language"] = lang; + tdi["compiler-id"] = + this->Makefile->GetSafeDefinition("CMAKE_" + lang + "_COMPILER_ID"); + + if (lang == "Fortran") { + std::string mod_dir = this->GeneratorTarget->GetFortranModuleDirectory( + this->Makefile->GetHomeOutputDirectory()); + if (mod_dir.empty()) { + mod_dir = this->Makefile->GetCurrentBinaryDirectory(); + } + tdi["module-dir"] = mod_dir; + } + + tdi["dir-cur-bld"] = this->Makefile->GetCurrentBinaryDirectory(); + tdi["dir-cur-src"] = this->Makefile->GetCurrentSourceDirectory(); + tdi["dir-top-bld"] = this->Makefile->GetHomeOutputDirectory(); + tdi["dir-top-src"] = this->Makefile->GetHomeDirectory(); + + Json::Value& tdi_include_dirs = tdi["include-dirs"] = Json::arrayValue; + std::vector includes; + this->LocalGenerator->GetIncludeDirectories(includes, this->GeneratorTarget, + lang, this->GetConfigName()); + for (std::vector::iterator i = includes.begin(); + i != includes.end(); ++i) { + tdi_include_dirs.append(*i); + } + + Json::Value& tdi_linked_target_dirs = tdi["linked-target-dirs"] = + Json::arrayValue; + std::vector linked = this->GetLinkedTargetDirectories(); + for (std::vector::iterator i = linked.begin(); + i != linked.end(); ++i) { + tdi_linked_target_dirs.append(*i); + } + + std::string const tdin = this->GetTargetDependInfoPath(lang); + cmGeneratedFileStream tdif(tdin.c_str()); + tdif << tdi; +} + void cmNinjaTargetGenerator::ExportObjectCompileCommand( std::string const& language, std::string const& sourceFileName, std::string const& objectDir, std::string const& objectFileName, diff --git a/Source/cmNinjaTargetGenerator.h b/Source/cmNinjaTargetGenerator.h index e6816db86..f5e6511e5 100644 --- a/Source/cmNinjaTargetGenerator.h +++ b/Source/cmNinjaTargetGenerator.h @@ -72,6 +72,8 @@ protected: std::string LanguageCompilerRule(const std::string& lang) const; std::string LanguagePreprocessRule(std::string const& lang) const; bool NeedExplicitPreprocessing(std::string const& lang) const; + std::string LanguageDyndepRule(std::string const& lang) const; + bool NeedDyndep(std::string const& lang) const; std::string OrderDependsTargetForTarget(); @@ -112,6 +114,12 @@ protected: /// @return the preprocessed source file path for the given @a source. std::string GetPreprocessedFilePath(cmSourceFile const* source) const; + /// @return the dyndep file path for this target. + std::string GetDyndepFilePath(std::string const& lang) const; + + /// @return the target dependency scanner info file path + std::string GetTargetDependInfoPath(std::string const& lang) const; + /// @return the file path where the target named @a name is generated. std::string GetTargetFilePath(const std::string& name) const; @@ -123,6 +131,7 @@ protected: void WriteObjectBuildStatements(); void WriteObjectBuildStatement(cmSourceFile const* source, bool writeOrderDependsTargetForTarget); + void WriteTargetDependInfo(std::string const& lang); void ExportObjectCompileCommand( std::string const& language, std::string const& sourceFileName, @@ -166,6 +175,7 @@ private: cmLocalNinjaGenerator* LocalGenerator; /// List of object files for this target. cmNinjaDeps Objects; + cmNinjaDeps DDIFiles; // TODO: Make per-language. std::vector CustomCommands; cmNinjaDeps ExtraFiles; }; From 330581502a7c7332a8f3ec15d57e1c21f7ff1ca8 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 22 Sep 2016 10:56:56 -0400 Subject: [PATCH 10/10] Help: Document Ninja generator conditional Fortran support Closes: #14215 --- Help/generator/Ninja.rst | 10 ++++++++++ Help/release/dev/ninja-fortran.rst | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 Help/release/dev/ninja-fortran.rst diff --git a/Help/generator/Ninja.rst b/Help/generator/Ninja.rst index ef0e28be0..3bbd9dcbe 100644 --- a/Help/generator/Ninja.rst +++ b/Help/generator/Ninja.rst @@ -21,3 +21,13 @@ are generated: ``sub/dir/package`` Runs the package step in the subdirectory, if any. + +Fortran Support +^^^^^^^^^^^^^^^ + +The ``Ninja`` generator conditionally supports Fortran when the ``ninja`` +tool has the required features. As of this version of CMake the needed +features have not been integrated into upstream Ninja. Kitware maintains +a branch of Ninja with the required features on `github.com/Kitware/ninja`_. + +.. _`github.com/Kitware/ninja`: https://github.com/Kitware/ninja/tree/features-for-fortran#readme diff --git a/Help/release/dev/ninja-fortran.rst b/Help/release/dev/ninja-fortran.rst new file mode 100644 index 000000000..612b1ff81 --- /dev/null +++ b/Help/release/dev/ninja-fortran.rst @@ -0,0 +1,6 @@ +ninja-fortran +------------- + +* The :generator:`Ninja` generator learned to conditionally support + Fortran when using a ``ninja`` tool that has the necessary features. + See generator documentation for details.