From 8521fdf56e4908676c28c6bbdda3f1fb2284d3d7 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 5 Feb 2015 16:48:16 -0500 Subject: [PATCH] Makefile: Fix output during parallel builds (#12991) Replace use of separate "cmake -E cmake_progress_report" and "cmake -E cmake_echo_color" commands to report the progress and message portions of build output lines with --progress-* options to the latter to print everything with a single command. The line buffering of the stdout FILE stream should cause the whole line to be printed with one atomic write. This will avoid inter-mixing of line-wise messages from different processes during a parallel build. --- Source/cmGlobalUnixMakefileGenerator3.cxx | 43 ++++---- Source/cmLocalUnixMakefileGenerator3.cxx | 23 ++++- Source/cmLocalUnixMakefileGenerator3.h | 5 +- Source/cmMakefileTargetGenerator.cxx | 36 ++++--- Source/cmMakefileTargetGenerator.h | 2 +- Source/cmcmd.cxx | 117 ++++++++++++++-------- 6 files changed, 133 insertions(+), 93 deletions(-) diff --git a/Source/cmGlobalUnixMakefileGenerator3.cxx b/Source/cmGlobalUnixMakefileGenerator3.cxx index 5f1bb831d..e0ccaa99d 100644 --- a/Source/cmGlobalUnixMakefileGenerator3.cxx +++ b/Source/cmGlobalUnixMakefileGenerator3.cxx @@ -779,29 +779,24 @@ cmGlobalUnixMakefileGenerator3 localName += "/all"; depends.clear(); - std::string progressDir = - lg->GetMakefile()->GetHomeOutputDirectory(); - progressDir += cmake::GetCMakeFilesDirectory(); + cmLocalUnixMakefileGenerator3::EchoProgress progress; + progress.Dir = lg->GetMakefile()->GetHomeOutputDirectory(); + progress.Dir += cmake::GetCMakeFilesDirectory(); + { + std::ostringstream progressArg; + const char* sep = ""; + std::vector& progFiles = + this->ProgressMap[gtarget->Target].Marks; + for (std::vector::iterator i = progFiles.begin(); + i != progFiles.end(); ++i) { - std::ostringstream progCmd; - progCmd << "$(CMAKE_COMMAND) -E cmake_progress_report "; - // all target counts - progCmd << lg->Convert(progressDir, - cmLocalGenerator::FULL, - cmLocalGenerator::SHELL); - progCmd << " "; - std::vector& progFiles = - this->ProgressMap[gtarget->Target].Marks; - for (std::vector::iterator i = progFiles.begin(); - i != progFiles.end(); ++i) - { - progCmd << " " << *i; - } - commands.push_back(progCmd.str()); + progressArg << sep << *i; + sep = ","; } - progressDir = "Built target "; - progressDir += name; - lg->AppendEcho(commands,progressDir.c_str()); + progress.Arg = progressArg.str(); + } + lg->AppendEcho(commands, "Built target " + name, + cmLocalUnixMakefileGenerator3::EchoNormal, &progress); this->AppendGlobalTargetDepends(depends,*gtarget->Target); lg->WriteMakeRule(ruleFileStream, "All Build rule for target.", @@ -819,15 +814,13 @@ cmGlobalUnixMakefileGenerator3 // Write the rule. commands.clear(); - progressDir = lg->GetMakefile()->GetHomeOutputDirectory(); - progressDir += cmake::GetCMakeFilesDirectory(); { // TODO: Convert the total progress count to a make variable. std::ostringstream progCmd; progCmd << "$(CMAKE_COMMAND) -E cmake_progress_start "; // # in target - progCmd << lg->Convert(progressDir, + progCmd << lg->Convert(progress.Dir, cmLocalGenerator::FULL, cmLocalGenerator::SHELL); // @@ -843,7 +836,7 @@ cmGlobalUnixMakefileGenerator3 { std::ostringstream progCmd; progCmd << "$(CMAKE_COMMAND) -E cmake_progress_start "; // # 0 - progCmd << lg->Convert(progressDir, + progCmd << lg->Convert(progress.Dir, cmLocalGenerator::FULL, cmLocalGenerator::SHELL); progCmd << " 0"; diff --git a/Source/cmLocalUnixMakefileGenerator3.cxx b/Source/cmLocalUnixMakefileGenerator3.cxx index 72d4ef072..c60a9c740 100644 --- a/Source/cmLocalUnixMakefileGenerator3.cxx +++ b/Source/cmLocalUnixMakefileGenerator3.cxx @@ -1346,8 +1346,9 @@ cmLocalUnixMakefileGenerator3 //---------------------------------------------------------------------------- void cmLocalUnixMakefileGenerator3::AppendEcho(std::vector& commands, - const char* text, - EchoColor color) + std::string const& text, + EchoColor color, + EchoProgress const* progress) { // Choose the color for the text. std::string color_name; @@ -1380,7 +1381,7 @@ cmLocalUnixMakefileGenerator3::AppendEcho(std::vector& commands, // Echo one line at a time. std::string line; line.reserve(200); - for(const char* c = text;; ++c) + for(const char* c = text.c_str();; ++c) { if(*c == '\n' || *c == '\0') { @@ -1389,7 +1390,7 @@ cmLocalUnixMakefileGenerator3::AppendEcho(std::vector& commands, { // Add a command to echo this line. std::string cmd; - if(color_name.empty()) + if(color_name.empty() && !progress) { // Use the native echo command. cmd = "@echo "; @@ -1400,6 +1401,17 @@ cmLocalUnixMakefileGenerator3::AppendEcho(std::vector& commands, // Use cmake to echo the text in color. cmd = "@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) "; cmd += color_name; + if (progress) + { + cmd += "--progress-dir="; + cmd += this->Convert(progress->Dir, + cmLocalGenerator::FULL, + cmLocalGenerator::SHELL); + cmd += " "; + cmd += "--progress-num="; + cmd += progress->Arg; + cmd += " "; + } cmd += this->EscapeForShell(line); } commands.push_back(cmd); @@ -1408,6 +1420,9 @@ cmLocalUnixMakefileGenerator3::AppendEcho(std::vector& commands, // Reset the line to emtpy. line = ""; + // Progress appears only on first line. + progress = 0; + // Terminate on end-of-string. if(*c == '\0') { diff --git a/Source/cmLocalUnixMakefileGenerator3.h b/Source/cmLocalUnixMakefileGenerator3.h index 7c8e27f49..1ff5e7fea 100644 --- a/Source/cmLocalUnixMakefileGenerator3.h +++ b/Source/cmLocalUnixMakefileGenerator3.h @@ -184,8 +184,9 @@ public: // append an echo command enum EchoColor { EchoNormal, EchoDepend, EchoBuild, EchoLink, EchoGenerate, EchoGlobal }; - void AppendEcho(std::vector& commands, const char* text, - EchoColor color = EchoNormal); + struct EchoProgress { std::string Dir; std::string Arg; }; + void AppendEcho(std::vector& commands, std::string const& text, + EchoColor color = EchoNormal, EchoProgress const* = 0); /** Get whether the makefile is to have color. */ bool GetColorMakefile() const { return this->ColorMakefile; } diff --git a/Source/cmMakefileTargetGenerator.cxx b/Source/cmMakefileTargetGenerator.cxx index 7ed0c106a..20207f50b 100644 --- a/Source/cmMakefileTargetGenerator.cxx +++ b/Source/cmMakefileTargetGenerator.cxx @@ -618,16 +618,19 @@ cmMakefileTargetGenerator std::vector commands; // add in a progress call if needed - this->AppendProgress(commands); + this->NumberOfProgressActions++; if(!this->NoRuleMessages) { + cmLocalUnixMakefileGenerator3::EchoProgress progress; + this->MakeEchoProgress(progress); std::string buildEcho = "Building "; buildEcho += lang; buildEcho += " object "; buildEcho += relativeObj; this->LocalGenerator->AppendEcho - (commands, buildEcho.c_str(), cmLocalUnixMakefileGenerator3::EchoBuild); + (commands, buildEcho.c_str(), cmLocalUnixMakefileGenerator3::EchoBuild, + &progress); } std::string targetOutPathReal; @@ -1200,12 +1203,15 @@ void cmMakefileTargetGenerator if(!comment.empty()) { // add in a progress call if needed - this->AppendProgress(commands); + this->NumberOfProgressActions++; if(!this->NoRuleMessages) { + cmLocalUnixMakefileGenerator3::EchoProgress progress; + this->MakeEchoProgress(progress); this->LocalGenerator ->AppendEcho(commands, comment.c_str(), - cmLocalUnixMakefileGenerator3::EchoGenerate); + cmLocalUnixMakefileGenerator3::EchoGenerate, + &progress); } } @@ -1263,22 +1269,14 @@ void cmMakefileTargetGenerator //---------------------------------------------------------------------------- void -cmMakefileTargetGenerator::AppendProgress(std::vector& commands) +cmMakefileTargetGenerator +::MakeEchoProgress(cmLocalUnixMakefileGenerator3::EchoProgress& progress) const { - this->NumberOfProgressActions++; - if(this->NoRuleMessages) - { - return; - } - std::string progressDir = this->Makefile->GetHomeOutputDirectory(); - progressDir += cmake::GetCMakeFilesDirectory(); - std::ostringstream progCmd; - progCmd << "$(CMAKE_COMMAND) -E cmake_progress_report "; - progCmd << this->LocalGenerator->Convert(progressDir, - cmLocalGenerator::FULL, - cmLocalGenerator::SHELL); - progCmd << " $(CMAKE_PROGRESS_" << this->NumberOfProgressActions << ")"; - commands.push_back(progCmd.str()); + progress.Dir = this->Makefile->GetHomeOutputDirectory(); + progress.Dir += cmake::GetCMakeFilesDirectory(); + std::ostringstream progressArg; + progressArg << "$(CMAKE_PROGRESS_" << this->NumberOfProgressActions << ")"; + progress.Arg = progressArg.str(); } //---------------------------------------------------------------------------- diff --git a/Source/cmMakefileTargetGenerator.h b/Source/cmMakefileTargetGenerator.h index e31e0863a..b072cfaea 100644 --- a/Source/cmMakefileTargetGenerator.h +++ b/Source/cmMakefileTargetGenerator.h @@ -109,7 +109,7 @@ protected: void GenerateExtraOutput(const char* out, const char* in, bool symbolic = false); - void AppendProgress(std::vector& commands); + void MakeEchoProgress(cmLocalUnixMakefileGenerator3::EchoProgress&) const; // write out the variable that lists the objects for this target void WriteObjectsVariable(std::string& variableName, diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index 6b04d260e..5260cb0e8 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -534,48 +534,9 @@ int cmcmd::ExecuteCMakeCommand(std::vector& args) // Command to report progress for a build else if (args[1] == "cmake_progress_report" && args.size() >= 3) { - std::string dirName = args[2]; - dirName += "/Progress"; - std::string fName; - FILE *progFile; - - // read the count - fName = dirName; - fName += "/count.txt"; - progFile = cmsys::SystemTools::Fopen(fName,"r"); - int count = 0; - if (!progFile) - { - return 0; - } - else - { - if (1!=fscanf(progFile,"%i",&count)) - { - cmSystemTools::Message("Could not read from progress file."); - } - fclose(progFile); - } - unsigned int i; - for (i = 3; i < args.size(); ++i) - { - fName = dirName; - fName += "/"; - fName += args[i]; - progFile = cmsys::SystemTools::Fopen(fName,"w"); - if (progFile) - { - fprintf(progFile,"empty"); - fclose(progFile); - } - } - int fileNum = static_cast - (cmsys::Directory::GetNumberOfFilesInDirectory(dirName)); - if (count > 0) - { - // print the progress - fprintf(stdout,"[%3i%%] ",((fileNum-3)*100)/count); - } + // This has been superseded by cmake_echo_color --progress-* + // options. We leave it here to avoid errors if somehow this + // is invoked by an existing makefile without regenerating. return 0; } @@ -986,6 +947,65 @@ bool cmcmd::SymlinkInternal(std::string const& file, std::string const& link) #endif } +//---------------------------------------------------------------------------- +static void cmcmdProgressReport(std::string const& dir, + std::string const& num) +{ + std::string dirName = dir; + dirName += "/Progress"; + std::string fName; + FILE *progFile; + + // read the count + fName = dirName; + fName += "/count.txt"; + progFile = cmsys::SystemTools::Fopen(fName,"r"); + int count = 0; + if (!progFile) + { + return; + } + else + { + if (1!=fscanf(progFile,"%i",&count)) + { + cmSystemTools::Message("Could not read from progress file."); + } + fclose(progFile); + } + const char* last = num.c_str(); + for(const char* c = last;; ++c) + { + if (*c == ',' || *c == '\0') + { + if (c != last) + { + fName = dirName; + fName += "/"; + fName.append(last, c-last); + progFile = cmsys::SystemTools::Fopen(fName,"w"); + if (progFile) + { + fprintf(progFile,"empty"); + fclose(progFile); + } + } + if(*c == '\0') + { + break; + } + last = c + 1; + } + } + int fileNum = static_cast + (cmsys::Directory::GetNumberOfFilesInDirectory(dirName)); + if (count > 0) + { + // print the progress + fprintf(stdout,"[%3i%%] ",((fileNum-3)*100)/count); + } +} + //---------------------------------------------------------------------------- int cmcmd::ExecuteEchoColor(std::vector& args) { @@ -996,6 +1016,7 @@ int cmcmd::ExecuteEchoColor(std::vector& args) bool enabled = true; int color = cmsysTerminal_Color_Normal; bool newline = true; + std::string progressDir; for(unsigned int i=2; i < args.size(); ++i) { if(args[i].find("--switch=") == 0) @@ -1014,6 +1035,18 @@ int cmcmd::ExecuteEchoColor(std::vector& args) } } } + else if(cmHasLiteralPrefix(args[i], "--progress-dir=")) + { + progressDir = args[i].substr(15); + } + else if(cmHasLiteralPrefix(args[i], "--progress-num=")) + { + if (!progressDir.empty()) + { + std::string const& progressNum = args[i].substr(15); + cmcmdProgressReport(progressDir, progressNum); + } + } else if(args[i] == "--normal") { color = cmsysTerminal_Color_Normal;