Major optimization of C/C++ dependency scanning.
Now only the dependencies for the file where the dependencies actually may have changed are rescanned, before that this was done for all source files even if only one source file had changed. This reduces e.g. on my machine the time for scanning the dependencies of kdelibs/khtml/ when only one file (khtml_global.cpp) has changed from around 7.5 seconds to 1.2 seconds. The tests succeed, it does what I expected it to do on kdelibs, and Brad also reviewed the patch, so I think it should be ok. Alex
This commit is contained in:
parent
551fcc23c2
commit
39383ef8cb
|
@ -87,7 +87,8 @@ bool cmDepends::Finalize(std::ostream&,
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
bool cmDepends::Check(const char *makeFile, const char *internalFile)
|
bool cmDepends::Check(const char *makeFile, const char *internalFile,
|
||||||
|
std::map<std::string, DependencyVector>& validDeps)
|
||||||
{
|
{
|
||||||
// Dependency checks must be done in proper working directory.
|
// Dependency checks must be done in proper working directory.
|
||||||
std::string oldcwd = ".";
|
std::string oldcwd = ".";
|
||||||
|
@ -102,7 +103,7 @@ bool cmDepends::Check(const char *makeFile, const char *internalFile)
|
||||||
// Check whether dependencies must be regenerated.
|
// Check whether dependencies must be regenerated.
|
||||||
bool okay = true;
|
bool okay = true;
|
||||||
std::ifstream fin(internalFile);
|
std::ifstream fin(internalFile);
|
||||||
if(!(fin && this->CheckDependencies(fin)))
|
if(!(fin && this->CheckDependencies(fin, validDeps)))
|
||||||
{
|
{
|
||||||
// Clear all dependencies so they will be regenerated.
|
// Clear all dependencies so they will be regenerated.
|
||||||
this->Clear(makeFile);
|
this->Clear(makeFile);
|
||||||
|
@ -146,13 +147,16 @@ bool cmDepends::WriteDependencies(const char*, const char*,
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
bool cmDepends::CheckDependencies(std::istream& internalDepends)
|
bool cmDepends::CheckDependencies(std::istream& internalDepends,
|
||||||
|
std::map<std::string, DependencyVector>& validDeps)
|
||||||
{
|
{
|
||||||
// Parse dependencies from the stream. If any dependee is missing
|
// Parse dependencies from the stream. If any dependee is missing
|
||||||
// or newer than the depender then dependencies should be
|
// or newer than the depender then dependencies should be
|
||||||
// regenerated.
|
// regenerated.
|
||||||
bool okay = true;
|
bool okay = true;
|
||||||
bool dependerExists = false;
|
bool dependerExists = false;
|
||||||
|
DependencyVector* currentDependencies = 0;
|
||||||
|
|
||||||
while(internalDepends.getline(this->Dependee, this->MaxPath))
|
while(internalDepends.getline(this->Dependee, this->MaxPath))
|
||||||
{
|
{
|
||||||
if ( this->Dependee[0] == 0 || this->Dependee[0] == '#' ||
|
if ( this->Dependee[0] == 0 || this->Dependee[0] == '#' ||
|
||||||
|
@ -174,6 +178,9 @@ bool cmDepends::CheckDependencies(std::istream& internalDepends)
|
||||||
// kdelibs/khtml this reduces the number of calls from 184k down to 92k,
|
// kdelibs/khtml this reduces the number of calls from 184k down to 92k,
|
||||||
// or the time for cmake -E cmake_depends from 0.3 s down to 0.21 s.
|
// or the time for cmake -E cmake_depends from 0.3 s down to 0.21 s.
|
||||||
dependerExists = cmSystemTools::FileExists(this->Depender);
|
dependerExists = cmSystemTools::FileExists(this->Depender);
|
||||||
|
DependencyVector tmp;
|
||||||
|
validDeps[this->Depender] = tmp;
|
||||||
|
currentDependencies = &validDeps[this->Depender];
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
|
@ -189,6 +196,11 @@ bool cmDepends::CheckDependencies(std::istream& internalDepends)
|
||||||
bool regenerate = false;
|
bool regenerate = false;
|
||||||
const char* dependee = this->Dependee+1;
|
const char* dependee = this->Dependee+1;
|
||||||
const char* depender = this->Depender;
|
const char* depender = this->Depender;
|
||||||
|
if (currentDependencies != 0)
|
||||||
|
{
|
||||||
|
currentDependencies->push_back(dependee);
|
||||||
|
}
|
||||||
|
|
||||||
if(!cmSystemTools::FileExists(dependee))
|
if(!cmSystemTools::FileExists(dependee))
|
||||||
{
|
{
|
||||||
// The dependee does not exist.
|
// The dependee does not exist.
|
||||||
|
@ -230,6 +242,14 @@ bool cmDepends::CheckDependencies(std::istream& internalDepends)
|
||||||
// Dependencies must be regenerated.
|
// Dependencies must be regenerated.
|
||||||
okay = false;
|
okay = false;
|
||||||
|
|
||||||
|
// Remove the information of this depender from the map, it needs
|
||||||
|
// to be rescanned
|
||||||
|
if (currentDependencies != 0)
|
||||||
|
{
|
||||||
|
validDeps.erase(this->Depender);
|
||||||
|
currentDependencies = 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Remove the depender to be sure it is rebuilt.
|
// Remove the depender to be sure it is rebuilt.
|
||||||
if (dependerExists)
|
if (dependerExists)
|
||||||
{
|
{
|
||||||
|
|
|
@ -60,11 +60,15 @@ public:
|
||||||
/** Write dependencies for the target file. */
|
/** Write dependencies for the target file. */
|
||||||
bool Write(std::ostream &makeDepends, std::ostream &internalDepends);
|
bool Write(std::ostream &makeDepends, std::ostream &internalDepends);
|
||||||
|
|
||||||
|
class DependencyVector: public std::vector<std::string> {};
|
||||||
|
|
||||||
/** Check dependencies for the target file. Returns true if
|
/** Check dependencies for the target file. Returns true if
|
||||||
dependencies are okay and false if they must be generated. If
|
dependencies are okay and false if they must be generated. If
|
||||||
they must be generated Clear has already been called to wipe out
|
they must be generated Clear has already been called to wipe out
|
||||||
the old dependencies. */
|
the old dependencies.
|
||||||
bool Check(const char *makeFile, const char* internalFile);
|
Dependencies which are still valid will be stored in validDeps. */
|
||||||
|
bool Check(const char *makeFile, const char* internalFile,
|
||||||
|
std::map<std::string, DependencyVector>& validDeps);
|
||||||
|
|
||||||
/** Clear dependencies for the target file so they will be regenerated. */
|
/** Clear dependencies for the target file so they will be regenerated. */
|
||||||
void Clear(const char *file);
|
void Clear(const char *file);
|
||||||
|
@ -83,7 +87,8 @@ protected:
|
||||||
// Check dependencies for the target file in the given stream.
|
// Check dependencies for the target file in the given stream.
|
||||||
// Return false if dependencies must be regenerated and true
|
// Return false if dependencies must be regenerated and true
|
||||||
// otherwise.
|
// otherwise.
|
||||||
virtual bool CheckDependencies(std::istream& internalDepends);
|
virtual bool CheckDependencies(std::istream& internalDepends,
|
||||||
|
std::map<std::string, DependencyVector>& validDeps);
|
||||||
|
|
||||||
// Finalize the dependency information for the target.
|
// Finalize the dependency information for the target.
|
||||||
virtual bool Finalize(std::ostream& makeDepends,
|
virtual bool Finalize(std::ostream& makeDepends,
|
||||||
|
|
|
@ -34,12 +34,17 @@
|
||||||
|
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
cmDependsC::cmDependsC()
|
cmDependsC::cmDependsC()
|
||||||
|
: ValidDeps(0)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
cmDependsC::cmDependsC(cmLocalGenerator* lg, const char* targetDir,
|
cmDependsC::cmDependsC(cmLocalGenerator* lg,
|
||||||
const char* lang): cmDepends(lg, targetDir)
|
const char* targetDir,
|
||||||
|
const char* lang,
|
||||||
|
const std::map<std::string, DependencyVector>* validDeps)
|
||||||
|
: cmDepends(lg, targetDir)
|
||||||
|
, ValidDeps(validDeps)
|
||||||
{
|
{
|
||||||
cmMakefile* mf = lg->GetMakefile();
|
cmMakefile* mf = lg->GetMakefile();
|
||||||
|
|
||||||
|
@ -113,6 +118,32 @@ bool cmDependsC::WriteDependencies(const char *src, const char *obj,
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this->ValidDeps != 0)
|
||||||
|
{
|
||||||
|
std::map<std::string, DependencyVector>::const_iterator tmpIt =
|
||||||
|
this->ValidDeps->find(obj);
|
||||||
|
if (tmpIt!= this->ValidDeps->end())
|
||||||
|
{
|
||||||
|
// Write the dependencies to the output stream. Makefile rules
|
||||||
|
// written by the original local generator for this directory
|
||||||
|
// convert the dependencies to paths relative to the home output
|
||||||
|
// directory. We must do the same here.
|
||||||
|
internalDepends << obj << std::endl;
|
||||||
|
for(DependencyVector::const_iterator i=tmpIt->second.begin();
|
||||||
|
i != tmpIt->second.end(); ++i)
|
||||||
|
{
|
||||||
|
makeDepends << obj << ": " <<
|
||||||
|
this->LocalGenerator->Convert(i->c_str(),
|
||||||
|
cmLocalGenerator::HOME_OUTPUT,
|
||||||
|
cmLocalGenerator::MAKEFILE)
|
||||||
|
<< std::endl;
|
||||||
|
internalDepends << " " << i->c_str() << std::endl;
|
||||||
|
}
|
||||||
|
makeDepends << std::endl;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Walk the dependency graph starting with the source file.
|
// Walk the dependency graph starting with the source file.
|
||||||
bool first = true;
|
bool first = true;
|
||||||
UnscannedEntry root;
|
UnscannedEntry root;
|
||||||
|
|
|
@ -30,7 +30,8 @@ public:
|
||||||
/** Checking instances need to know the build directory name and the
|
/** Checking instances need to know the build directory name and the
|
||||||
relative path from the build directory to the target file. */
|
relative path from the build directory to the target file. */
|
||||||
cmDependsC();
|
cmDependsC();
|
||||||
cmDependsC(cmLocalGenerator* lg, const char* targetDir, const char* lang);
|
cmDependsC(cmLocalGenerator* lg, const char* targetDir, const char* lang,
|
||||||
|
const std::map<std::string, DependencyVector>* validDeps);
|
||||||
|
|
||||||
/** Virtual destructor to cleanup subclasses properly. */
|
/** Virtual destructor to cleanup subclasses properly. */
|
||||||
virtual ~cmDependsC();
|
virtual ~cmDependsC();
|
||||||
|
@ -83,6 +84,7 @@ public:
|
||||||
bool Used;
|
bool Used;
|
||||||
};
|
};
|
||||||
protected:
|
protected:
|
||||||
|
const std::map<std::string, DependencyVector>* ValidDeps;
|
||||||
std::set<cmStdString> Encountered;
|
std::set<cmStdString> Encountered;
|
||||||
std::queue<UnscannedEntry> Unscanned;
|
std::queue<UnscannedEntry> Unscanned;
|
||||||
t_CharBuffer Buffer;
|
t_CharBuffer Buffer;
|
||||||
|
|
|
@ -43,7 +43,8 @@ bool cmDependsJava::WriteDependencies(const char *src, const char *,
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool cmDependsJava::CheckDependencies(std::istream&)
|
bool cmDependsJava::CheckDependencies(std::istream&,
|
||||||
|
std::map<std::string, DependencyVector >&)
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,8 @@ protected:
|
||||||
// Implement writing/checking methods required by superclass.
|
// Implement writing/checking methods required by superclass.
|
||||||
virtual bool WriteDependencies(const char *src, const char *file,
|
virtual bool WriteDependencies(const char *src, const char *file,
|
||||||
std::ostream& makeDepends, std::ostream& internalDepends);
|
std::ostream& makeDepends, std::ostream& internalDepends);
|
||||||
virtual bool CheckDependencies(std::istream& internalDepends);
|
virtual bool CheckDependencies(std::istream& internalDepends,
|
||||||
|
std::map<std::string, DependencyVector >& validDeps);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
cmDependsJava(cmDependsJava const&); // Purposely not implemented.
|
cmDependsJava(cmDependsJava const&); // Purposely not implemented.
|
||||||
|
|
|
@ -16,7 +16,6 @@
|
||||||
=========================================================================*/
|
=========================================================================*/
|
||||||
#include "cmLocalUnixMakefileGenerator3.h"
|
#include "cmLocalUnixMakefileGenerator3.h"
|
||||||
|
|
||||||
#include "cmDepends.h"
|
|
||||||
#include "cmGeneratedFileStream.h"
|
#include "cmGeneratedFileStream.h"
|
||||||
#include "cmGlobalUnixMakefileGenerator3.h"
|
#include "cmGlobalUnixMakefileGenerator3.h"
|
||||||
#include "cmMakefile.h"
|
#include "cmMakefile.h"
|
||||||
|
@ -1387,9 +1386,9 @@ bool cmLocalUnixMakefileGenerator3::UpdateDependencies(const char* tgtInfo,
|
||||||
if(verbose)
|
if(verbose)
|
||||||
{
|
{
|
||||||
cmOStringStream msg;
|
cmOStringStream msg;
|
||||||
msg << "Dependee \"" << internalDependFile
|
msg << "Dependee \"" << dirInfoFile
|
||||||
<< "\" is newer than depender \""
|
<< "\" is newer than depender \""
|
||||||
<< dirInfoFile << "\"." << std::endl;
|
<< internalDependFile << "\"." << std::endl;
|
||||||
cmSystemTools::Stdout(msg.str().c_str());
|
cmSystemTools::Stdout(msg.str().c_str());
|
||||||
}
|
}
|
||||||
needRescanDirInfo = true;
|
needRescanDirInfo = true;
|
||||||
|
@ -1400,14 +1399,26 @@ bool cmLocalUnixMakefileGenerator3::UpdateDependencies(const char* tgtInfo,
|
||||||
// The build.make file may have explicit dependencies for the object
|
// The build.make file may have explicit dependencies for the object
|
||||||
// files but these will not affect the scanning process so they need
|
// files but these will not affect the scanning process so they need
|
||||||
// not be considered.
|
// not be considered.
|
||||||
|
std::map<std::string, cmDepends::DependencyVector> validDependencies;
|
||||||
bool needRescanDependencies = false;
|
bool needRescanDependencies = false;
|
||||||
if (needRescanDirInfo == false)
|
if (needRescanDirInfo == false)
|
||||||
{
|
{
|
||||||
cmDependsC checker;
|
cmDependsC checker;
|
||||||
checker.SetVerbose(verbose);
|
checker.SetVerbose(verbose);
|
||||||
checker.SetFileComparison(ftc);
|
checker.SetFileComparison(ftc);
|
||||||
|
// cmDependsC::Check() fills the vector validDependencies() with the
|
||||||
|
// dependencies for those files where they are still valid, i.e. neither
|
||||||
|
// the files themselves nor any files they depend on have changed.
|
||||||
|
// We don't do that if the CMakeDirectoryInformation.cmake file has
|
||||||
|
// changed, because then potentially all dependencies have changed.
|
||||||
|
// This information is given later on to cmDependsC, which then only
|
||||||
|
// rescans the files where it did not get valid dependencies via this
|
||||||
|
// dependency vector. This means that in the normal case, when only
|
||||||
|
// few or one file have been edited, then also only this one file is
|
||||||
|
// actually scanned again, instead of all files for this target.
|
||||||
needRescanDependencies = !checker.Check(dependFile.c_str(),
|
needRescanDependencies = !checker.Check(dependFile.c_str(),
|
||||||
internalDependFile.c_str());
|
internalDependFile.c_str(),
|
||||||
|
validDependencies);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(needRescanDependInfo || needRescanDirInfo || needRescanDependencies)
|
if(needRescanDependInfo || needRescanDirInfo || needRescanDependencies)
|
||||||
|
@ -1426,7 +1437,7 @@ bool cmLocalUnixMakefileGenerator3::UpdateDependencies(const char* tgtInfo,
|
||||||
fprintf(stdout, "%s\n", message.c_str());
|
fprintf(stdout, "%s\n", message.c_str());
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return this->ScanDependencies(dir.c_str());
|
return this->ScanDependencies(dir.c_str(), validDependencies);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -1438,7 +1449,8 @@ bool cmLocalUnixMakefileGenerator3::UpdateDependencies(const char* tgtInfo,
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
bool
|
bool
|
||||||
cmLocalUnixMakefileGenerator3
|
cmLocalUnixMakefileGenerator3
|
||||||
::ScanDependencies(const char* targetDir)
|
::ScanDependencies(const char* targetDir,
|
||||||
|
std::map<std::string, cmDepends::DependencyVector>& validDeps)
|
||||||
{
|
{
|
||||||
// Read the directory information file.
|
// Read the directory information file.
|
||||||
cmMakefile* mf = this->Makefile;
|
cmMakefile* mf = this->Makefile;
|
||||||
|
@ -1526,7 +1538,7 @@ cmLocalUnixMakefileGenerator3
|
||||||
if(lang == "C" || lang == "CXX" || lang == "RC")
|
if(lang == "C" || lang == "CXX" || lang == "RC")
|
||||||
{
|
{
|
||||||
// TODO: Handle RC (resource files) dependencies correctly.
|
// TODO: Handle RC (resource files) dependencies correctly.
|
||||||
scanner = new cmDependsC(this, targetDir, lang.c_str());
|
scanner = new cmDependsC(this, targetDir, lang.c_str(), &validDeps);
|
||||||
}
|
}
|
||||||
#ifdef CMAKE_BUILD_WITH_CMAKE
|
#ifdef CMAKE_BUILD_WITH_CMAKE
|
||||||
else if(lang == "Fortran")
|
else if(lang == "Fortran")
|
||||||
|
|
|
@ -19,6 +19,9 @@
|
||||||
|
|
||||||
#include "cmLocalGenerator.h"
|
#include "cmLocalGenerator.h"
|
||||||
|
|
||||||
|
// for cmDepends::DependencyVector
|
||||||
|
#include "cmDepends.h"
|
||||||
|
|
||||||
class cmCustomCommand;
|
class cmCustomCommand;
|
||||||
class cmDependInformation;
|
class cmDependInformation;
|
||||||
class cmDepends;
|
class cmDepends;
|
||||||
|
@ -343,7 +346,8 @@ protected:
|
||||||
cmTarget& target, const char* filename =0);
|
cmTarget& target, const char* filename =0);
|
||||||
|
|
||||||
// Helper methods for dependeny updates.
|
// Helper methods for dependeny updates.
|
||||||
bool ScanDependencies(const char* targetDir);
|
bool ScanDependencies(const char* targetDir,
|
||||||
|
std::map<std::string, cmDepends::DependencyVector>& validDeps);
|
||||||
void CheckMultipleOutputs(bool verbose);
|
void CheckMultipleOutputs(bool verbose);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
Loading…
Reference in New Issue