From 7bdad5461477f8fdafafb850fb0b3e21c39fc556 Mon Sep 17 00:00:00 2001 From: Bill Hoffman Date: Fri, 1 Jun 2007 15:40:07 -0400 Subject: [PATCH] ENH: initial bullseye stuff --- Source/CTest/cmCTestCoverageHandler.cxx | 533 +++++++++++++++++++++++- Source/CTest/cmCTestCoverageHandler.h | 23 + 2 files changed, 549 insertions(+), 7 deletions(-) diff --git a/Source/CTest/cmCTestCoverageHandler.cxx b/Source/CTest/cmCTestCoverageHandler.cxx index 30e3633a3..441569d9a 100644 --- a/Source/CTest/cmCTestCoverageHandler.cxx +++ b/Source/CTest/cmCTestCoverageHandler.cxx @@ -1,6 +1,6 @@ /*========================================================================= - Program: CMake - Cross-Platform Makefile Generator + program: CMake - Cross-Platform Makefile Generator Module: $RCSfile$ Language: C++ Date: $Date$ @@ -14,9 +14,7 @@ PURPOSE. See the above copyright notices for more information. =========================================================================*/ - #include "cmCTestCoverageHandler.h" - #include "cmCTest.h" #include "cmake.h" #include "cmSystemTools.h" @@ -31,6 +29,96 @@ #include #define SAFEDIV(x,y) (((y)!=0)?((x)/(y)):(0)) +#include // remove me***** +#undef cerr + +class cmCTestRunProcess +{ +public: + cmCTestRunProcess() + { + this->Process = cmsysProcess_New(); + this->PipeState = -1; + } + ~cmCTestRunProcess() + { + if(!(this->PipeState == -1) + && !(this->PipeState == cmsysProcess_Pipe_None ) + && !(this->PipeState == cmsysProcess_Pipe_Timeout)) + { + this->WaitForExit(); + } + cmsysProcess_Delete(this->Process); + } + void SetCommand(const char* command) + { + this->CommandLineStrings.clear(); + this->CommandLineStrings.push_back(command);; + } + void AddArgument(const char* arg) + { + this->CommandLineStrings.push_back(arg); + } + void SetWorkingDirectory(const char* dir) + { + this->WorkingDirectory = dir; + } + void SetTimeout(double t) + { + this->TimeOut = t; + } + bool StartProcess() + { + std::vector args; + for(std::vector::iterator i = + this->CommandLineStrings.begin(); + i != this->CommandLineStrings.end(); ++i) + { + args.push_back(i->c_str()); + } + args.push_back(0); // null terminate + cmsysProcess_SetCommand(this->Process, &*args.begin()); + cmsysProcess_SetWorkingDirectory(this->Process, + this->WorkingDirectory.c_str()); + cmsysProcess_SetOption(this->Process, + cmsysProcess_Option_HideWindow, 1); + if(this->TimeOut != -1) + { + cmsysProcess_SetTimeout(this->Process, this->TimeOut); + } + cmsysProcess_Execute(this->Process); + this->PipeState = cmsysProcess_GetState(this->Process); + // if the process is running or exited return true + if(this->PipeState == cmsysProcess_State_Executing + || this->PipeState == cmsysProcess_State_Exited) + { + return true; + } + return false; + } + void SetStdoutFile(const char* fname) + { + cmsysProcess_SetPipeFile(this->Process, cmsysProcess_Pipe_STDOUT, fname); + } + void SetStderrFile(const char* fname) + { + cmsysProcess_SetPipeFile(this->Process, cmsysProcess_Pipe_STDERR, fname); + } + int WaitForExit(double* timeout =0) + { + this->PipeState = cmsysProcess_WaitForExit(this->Process, + timeout); + return this->PipeState; + } + int GetProcessState() { return this->PipeState;} +private: + int PipeState; + cmsysProcess* Process; + std::vector CommandLineStrings; + std::string WorkingDirectory; + double TimeOut; +}; + //---------------------------------------------------------------------- //********************************************************************** @@ -158,8 +246,16 @@ bool cmCTestCoverageHandler::ShouldIDoCoverage(const char* file, // By now checkDir should be set to parent directory of the file. // Get the relative path to the file an apply it to the opposite directory. // If it is the same as fileDir, then ignore, otherwise check. - std::string relPath = cmSystemTools::RelativePath(checkDir.c_str(), - fFile.c_str()); + std::string relPath; + if(checkDir.size() ) + { + relPath = cmSystemTools::RelativePath(checkDir.c_str(), + fFile.c_str()); + } + else + { + relPath = fFile; + } if ( checkDir == fSrcDir ) { checkDir = fBinDir; @@ -195,7 +291,8 @@ bool cmCTestCoverageHandler::ShouldIDoCoverage(const char* file, int cmCTestCoverageHandler::ProcessHandler() { int error = 0; - + cmCTestLog(this->CTest, ERROR_MESSAGE, + "ProcessHandler cmCTestCoverageHandler " << std::endl); // do we have time for this if (this->CTest->GetRemainingTimeAllowed() < 120) { @@ -233,8 +330,11 @@ int cmCTestCoverageHandler::ProcessHandler() cont.BinaryDir = binaryDir; cont.OFS = &ofs; + if(this->HandleBullseyeCoverage(&cont)) + { + return cont.Error; + } int file_count = 0; - file_count += this->HandleGCovCoverage(&cont); if ( file_count < 0 ) { @@ -1061,3 +1161,422 @@ std::string cmCTestCoverageHandler::FindFile( } return ""; } + +// This is a header put on each marked up source file +namespace +{ + const char* bullseyeHelp[] = + {" Coverage produced by bullseye covbr tool: ", + " www.bullseye.com/help/ref_covbr.html", + " * An arrow --> indicates incomplete coverage.", + " * An X indicates a function that was invoked, a switch label that ", + " was exercised, a try-block that finished, or an exception handler ", + " that was invoked.", + " * A T or F indicates a boolean decision that evaluated true or false,", + " respectively.", + " * A t or f indicates a boolean condition within a decision if the ", + " condition evaluated true or false, respectively.", + " * A k indicates a constant decision or condition.", + " * The slash / means this probe is excluded from summary results. ", + 0}; +} + +//---------------------------------------------------------------------- +int cmCTestCoverageHandler::RunBullseyeCoverageBranch( + cmCTestCoverageHandlerContainer* cont, + std::vector& files, + std::vector& filesFullPath) +{ + if(files.size() != filesFullPath.size()) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Files and full path files not the same size?:\n"); + return 0; + } + // create the output stream for the CoverageLog-N.xml file + cmGeneratedFileStream covLogFile; + int logFileCount = 0; + if ( !this->StartCoverageLogFile(covLogFile, logFileCount) ) + { + return -1; + } + int count =0; // keep count of the number of files + // loop over all files + std::vector::iterator fp = filesFullPath.begin(); + for(std::vector::iterator f = files.begin(); + f != files.end(); ++f, ++fp) + { + // only allow 100 files in each log file + if ( count != 0 && count % 100 == 0 ) + { + this->EndCoverageLogFile(covLogFile, logFileCount); + logFileCount ++; + if ( !this->StartCoverageLogFile(covLogFile, logFileCount) ) + { + return -1; + } + } + // for each file run covbr on that file to get the coverage + // information for that file + std::string outputFile; + if(!this->RunBullseyeCommand(cont, "covbr", f->c_str(), outputFile)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "error running covbr for:" << + f->c_str() << "\n"); + continue; + } + // open the output file + std::ifstream fin(outputFile.c_str()); + if(!fin) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Cannot open coverage file: " << + outputFile.c_str() << std::endl); + return 0; + } + // we have good data so we can count it + count++; + // start the file output + covLogFile << "\tCTest->MakeXMLSafe(f->c_str()) + << "\" FullPath=\"" << this->CTest->MakeXMLSafe( + this->CTest->GetShortPathToFile( + fp->c_str())) << "\">" << std::endl + << "\t\t" << std::endl; + // write the bullseye header + int line =0; + for(int k =0; bullseyeHelp[k] != 0; ++k) + { + covLogFile << "\t\t" + << this->CTest->MakeXMLSafe(bullseyeHelp[k]) + << "" << std::endl; + line++; + } + // Now copy each line from the bullseye file into the cov log file + std::string lineIn; + while(cmSystemTools::GetLineFromStream(fin, lineIn)) + { + covLogFile << "\t\t" + << this->CTest->MakeXMLSafe(lineIn.c_str()) + << "" << std::endl; + line++; + } + covLogFile << "\t\t" << std::endl + << "\t" << std::endl; + } + this->EndCoverageLogFile(covLogFile, logFileCount); + return 1; +} + +//---------------------------------------------------------------------- +int cmCTestCoverageHandler::RunBullseyeCommand( + cmCTestCoverageHandlerContainer* cont, + const char* cmd, + const char* arg, + std::string& outputFile) +{ + std::string program = cmSystemTools::FindProgram(cmd); + if(program.size() == 0) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot find :" << cmd << "\n"); + return 0; + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Run : " << program.c_str() << " " << arg << "\n"); + // create a process object and start it + cmCTestRunProcess runCoverageSrc; + runCoverageSrc.SetCommand(program.c_str()); + runCoverageSrc.AddArgument(arg); + std::string stdoutFile = cont->BinaryDir + "/Testing/Temporary/"; + stdoutFile += this->GetCTestInstance()->GetCurrentTag(); + stdoutFile + "-"; + stdoutFile += cmd; + std::string stderrFile = stdoutFile; + stdoutFile + ".stdout"; + stderrFile += ".stderr"; + runCoverageSrc.SetStdoutFile(stdoutFile.c_str()); + runCoverageSrc.SetStderrFile(stderrFile.c_str()); + if(!runCoverageSrc.StartProcess()) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Could not run : " + << program.c_str() << " " << arg << "\n" + << "kwsys process state : " + << runCoverageSrc.GetProcessState()); + return 0; + } + // since we set the output file names wait for it to end + runCoverageSrc.WaitForExit(); + outputFile = stdoutFile; + return 1; +} + +//---------------------------------------------------------------------- +int cmCTestCoverageHandler::RunBullseyeSourceSummary( + cmCTestCoverageHandlerContainer* cont) +{ + // Run the covsrc command and create a temp outputfile + std::string outputFile; + if(!this->RunBullseyeCommand(cont, "covsrc", "-c", outputFile)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "error running covsrc:\n"); + return 0; + } + + std::ostream& tmpLog = *cont->OFS; + // copen the Coverage.xml file in the Testing directory + cmGeneratedFileStream covSumFile; + if (!this->StartResultingXML("Coverage", covSumFile)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Cannot open coverage summary file." << std::endl); + return 0; + } + this->CTest->StartXML(covSumFile); + double elapsed_time_start = cmSystemTools::GetTime(); + std::string coverage_start_time = this->CTest->CurrentTime(); + covSumFile << "" << std::endl + << "\t" + << coverage_start_time << "" + << std::endl; + std::string stdline; + std::string errline; + // expected output: + // first line is: + // "Source","Function Coverage","out of","%","C/D Coverage","out of","%" + // after that data follows in that format + std::string sourceFile; + int functionsCalled = 0; + int totalFunctions = 0; + int percentFunction = 0; + int branchCovered = 0; + int totalBranches = 0; + int percentBranch = 0; + double total_tested = 0; + double total_untested = 0; + double total_functions = 0; + double percent_coverage =0; + double number_files = 0; + std::vector coveredFiles; + std::vector coveredFilesFullPath; + // Read and parse the summary output file + std::ifstream fin(outputFile.c_str()); + if(!fin) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Cannot open coverage summary file: " << + outputFile.c_str() << std::endl); + return 0; + } + while(cmSystemTools::GetLineFromStream(fin, stdline)) + { + // if we have a line of output from stdout + if(stdline.size()) + { + // parse the comma separated output + this->ParseBullsEyeCovsrcLine(stdline, + sourceFile, + functionsCalled, + totalFunctions, + percentFunction, + branchCovered, + totalBranches, + percentBranch); + // The first line is the header + if(sourceFile == "Source" || sourceFile == "Total") + { + continue; + } + std::string file = sourceFile; + if(!cmSystemTools::FileIsFullPath(sourceFile.c_str())) + { + // file will be relative to the binary dir + file = cont->BinaryDir; + file += "/"; + file += sourceFile; + } + file = cmSystemTools::CollapseFullPath(file.c_str()); + bool shouldIDoCoverage + = this->ShouldIDoCoverage(file.c_str(), + cont->SourceDir.c_str(), + cont->BinaryDir.c_str()); + if ( !shouldIDoCoverage ) + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + ".NoDartCoverage found, so skip coverage check for: " + << file.c_str() + << std::endl); + continue; + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Doing coverage for: " + << file.c_str() + << std::endl); + + coveredFiles.push_back(sourceFile); + number_files++; + total_functions += totalFunctions; + total_tested += functionsCalled; + total_untested += (totalFunctions - functionsCalled); + std::string fileName = cmSystemTools::GetFilenameName(file.c_str()); + // get file relative to the source dir + file = cmSystemTools::RelativePath(cont->SourceDir.c_str(), + file.c_str()); + coveredFilesFullPath.push_back(file); + float cper = percentBranch + percentFunction; + if(totalBranches > 0) + { + cper /= 2.0; + } + percent_coverage += cper; + float cmet = percentFunction + percentBranch; + if(totalBranches > 0) + { + cmet /= 2.0; + } + cmet /= 100.0; + // Hack for conversion of function to loc assume a function + // has 100 lines of code + functionsCalled *=100; + totalFunctions *=100; + tmpLog << stdline.c_str() << "\n"; + tmpLog << fileName << "\n"; + tmpLog << "functionsCalled: " << functionsCalled/100 << "\n"; + tmpLog << "totalFunctions: " << totalFunctions/100 << "\n"; + tmpLog << "percentFunction: " << percentFunction << "\n"; + tmpLog << "branchCovered: " << branchCovered << "\n"; + tmpLog << "totalBranches: " << totalBranches << "\n"; + tmpLog << "percentBranch: " << percentBranch << "\n"; + tmpLog << "percentCoverage: " << percent_coverage << "\n"; + tmpLog << "coverage metric: " << cmet << "\n"; + covSumFile << "\tCTest->MakeXMLSafe(fileName) + << "\" FullPath=\"" << this->CTest->MakeXMLSafe( + this->CTest->GetShortPathToFile(file.c_str())) + << "\" Covered=\"" << (cmet>0?"true":"false") << "\">\n" + << "\t\t" << functionsCalled << "\n" + << "\t\t" + << totalFunctions - functionsCalled << "\n" + << "\t\t"; + covSumFile.setf(std::ios::fixed, std::ios::floatfield); + covSumFile.precision(2); + covSumFile << (cper) << "\n" + << "\t\t"; + covSumFile.setf(std::ios::fixed, std::ios::floatfield); + covSumFile.precision(2); + covSumFile << (cmet) << "\n" + << "\t" << std::endl; + } + } + std::string end_time = this->CTest->CurrentTime(); + covSumFile << "\t" << total_tested << "\n" + << "\t" << total_untested << "\n" + << "\t" << total_functions << "\n" + << "\t"; + covSumFile.setf(std::ios::fixed, std::ios::floatfield); + covSumFile.precision(2); + covSumFile << (percent_coverage/number_files)<< "\n" + << "\t" << end_time << "\n"; + covSumFile << "" << + static_cast((cmSystemTools::GetTime() - elapsed_time_start)/6)/10.0 + << "" + << "" << std::endl; + this->CTest->EndXML(covSumFile); + // Now create the coverage information for each file + return this->RunBullseyeCoverageBranch(cont, + coveredFiles, + coveredFilesFullPath); +} + +//---------------------------------------------------------------------- +int cmCTestCoverageHandler::HandleBullseyeCoverage( + cmCTestCoverageHandlerContainer* cont) +{ + const char* covfile = cmSystemTools::GetEnv("COVFILE"); + if(!covfile) + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + " COVFILE environment variable not found, not running " + " bullseye\n"); + return 0; + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + " run covsrc with COVFILE=[" + << covfile + << "]" << std::endl); + if(!this->RunBullseyeSourceSummary(cont)) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error running bullseye summary.\n"); + return 0; + } + return 1; +} + +bool cmCTestCoverageHandler::GetNextInt(std::string const& inputLine, + std::string::size_type& pos, + int& value) +{ + std::string::size_type start = pos; + pos = inputLine.find(',', start); + value = atoi(inputLine.substr(start, pos).c_str()); + if(pos == inputLine.npos) + { + return true; + } + pos++; + return true; +} + +bool cmCTestCoverageHandler::ParseBullsEyeCovsrcLine( + std::string const& inputLine, + std::string& sourceFile, + int& functionsCalled, + int& totalFunctions, + int& percentFunction, + int& branchCovered, + int& totalBranches, + int& percentBranch) +{ + // find the first comma + std::string::size_type pos = inputLine.find(','); + if(pos == inputLine.npos) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Error parsing string : " + << inputLine.c_str() << "\n"); + return false; + } + // the source file has "" around it so extract out the file name + sourceFile = inputLine.substr(1,pos-2); + pos++; + if(!this->GetNextInt(inputLine, pos, functionsCalled)) + { + return false; + } + if(!this->GetNextInt(inputLine, pos, totalFunctions)) + { + return false; + } + if(!this->GetNextInt(inputLine, pos, percentFunction)) + { + return false; + } + if(!this->GetNextInt(inputLine, pos, branchCovered)) + { + return false; + } + if(!this->GetNextInt(inputLine, pos, totalBranches)) + { + return false; + } + if(!this->GetNextInt(inputLine, pos, percentBranch)) + { + return false; + } + // should be at the end now + if(pos != inputLine.npos) + { + cmCTestLog(this->CTest, ERROR_MESSAGE, "Error parsing input : " + << inputLine.c_str() << " last pos not npos = " << pos << + "\n"); + } + return true; +} diff --git a/Source/CTest/cmCTestCoverageHandler.h b/Source/CTest/cmCTestCoverageHandler.h index 1125768d1..99b85d3a5 100644 --- a/Source/CTest/cmCTestCoverageHandler.h +++ b/Source/CTest/cmCTestCoverageHandler.h @@ -59,6 +59,29 @@ private: //! Handle coverage using GCC's GCov int HandleGCovCoverage(cmCTestCoverageHandlerContainer* cont); + //! Handle coverage using Bullseye + int HandleBullseyeCoverage(cmCTestCoverageHandlerContainer* cont); + int RunBullseyeSourceSummary(cmCTestCoverageHandlerContainer* cont); + int RunBullseyeCoverageBranch(cmCTestCoverageHandlerContainer* cont, + std::vector& files, + std::vector& filesFullPath); + int RunBullseyeCommand( + cmCTestCoverageHandlerContainer* cont, + const char* cmd, + const char* arg, + std::string& outputFile); + bool ParseBullsEyeCovsrcLine( + std::string const& inputLine, + std::string& sourceFile, + int& functionsCalled, + int& totalFunctions, + int& percentFunction, + int& branchCovered, + int& totalBranches, + int& percentBranch); + bool GetNextInt(std::string const& inputLine, + std::string::size_type& pos, + int& value); //! Handle Python coverage using Python's Trace.py int HandleTracePyCoverage(cmCTestCoverageHandlerContainer* cont);