diff options
Diffstat (limited to 'comparison.cpp')
| -rw-r--r-- | comparison.cpp | 1156 |
1 files changed, 624 insertions, 532 deletions
diff --git a/comparison.cpp b/comparison.cpp index 843990a8..a43a0c86 100644 --- a/comparison.cpp +++ b/comparison.cpp @@ -1,4 +1,5 @@ #include "comparison.h" +#include <stdexcept> #include "shared/globalFunctions.h" #include <wx/intl.h> #include <wx/timer.h> @@ -13,238 +14,326 @@ #include "shared/systemFunctions.h" #include "shared/fileTraverser.h" #include "library/filter.h" +#include <map> +#include "fileHierarchy.h" +#include <boost/bind.hpp> using namespace FreeFileSync; -class GetAllFilesFull : public FreeFileSync::TraverseCallback +std::vector<FreeFileSync::FolderPairCfg> FreeFileSync::extractCompareCfg(const MainConfiguration& mainCfg) { + std::vector<FolderPairCfg> output; + + //add main pair + output.push_back( + FolderPairCfg(mainCfg.mainFolderPair.leftDirectory, + mainCfg.mainFolderPair.rightDirectory, + mainCfg.filterIsActive, + mainCfg.includeFilter, + mainCfg.excludeFilter, + mainCfg.syncConfiguration)); + + //add additional pairs + for (std::vector<FolderPairEnh>::const_iterator i = mainCfg.additionalPairs.begin(); i != mainCfg.additionalPairs.end(); ++i) + output.push_back( + FolderPairCfg(i->leftDirectory, + i->rightDirectory, + mainCfg.filterIsActive, + i->altFilter.get() ? i->altFilter->includeFilter : mainCfg.includeFilter, + i->altFilter.get() ? i->altFilter->excludeFilter : mainCfg.excludeFilter, + i->altSyncConfig.get() ? i->altSyncConfig->syncConfiguration : mainCfg.syncConfiguration)); + + return output; +} + + + + + + + +template <bool filterActive> +class BaseDirCallback; + + +template <bool filterActive> +class DirCallback : public FreeFileSync::TraverseCallback +{ +public: + DirCallback(BaseDirCallback<filterActive>* baseCallback, + const Zstring& relNameParentPf, //postfixed with FILE_NAME_SEPARATOR! + DirContainer& output, + StatusHandler* handler) : + baseCallback_(baseCallback), + relNameParentPf_(relNameParentPf), + output_(output), + statusHandler(handler) {} + + virtual ~DirCallback() {} + + virtual ReturnValue onFile(const DefaultChar* shortName, const Zstring& fullName, const FileInfo& details); + virtual ReturnValDir onDir(const DefaultChar* shortName, const Zstring& fullName); + virtual ReturnValue onError(const wxString& errorText); + +private: + BaseDirCallback<filterActive>* const baseCallback_; + const Zstring relNameParentPf_; + DirContainer& output_; + StatusHandler* const statusHandler; +}; + + + +template <bool filterActive> +class BaseDirCallback : public DirCallback<filterActive> +{ + friend class DirCallback<filterActive>; public: - GetAllFilesFull(DirectoryDescrType& output, const Zstring& dirThatIsSearched, const FilterProcess* filter, StatusHandler* handler) : - m_output(output), - directory(dirThatIsSearched), + BaseDirCallback(DirContainer& output, const FilterProcess* filter, StatusHandler* handler) : + DirCallback<filterActive>(this, Zstring(), output, handler), textScanning(Zstring(_("Scanning:")) + wxT(" \n")), - filterInstance(filter), - statusHandler(handler) - { - prefixLength = directory.length(); - } + filterInstance(filter) {} + virtual TraverseCallback::ReturnValDir onDir(const DefaultChar* shortName, const Zstring& fullName); - inline - void writeText(const wxChar* text, const int length, wxChar*& currentPos) - { - memcpy(currentPos, text, length * sizeof(wxChar)); - currentPos += length; - } +private: + typedef boost::shared_ptr<const DirCallback<filterActive> > CallbackPointer; + const Zstring textScanning; + std::vector<CallbackPointer> callBackBox; //collection of callback pointers to handle ownership + const FilterProcess* const filterInstance; //must NOT be NULL if (filterActive) +}; - virtual ReturnValue onFile(const DefaultChar* shortName, const Zstring& fullName, const FileInfo& details) - { - //apply filter before processing (use relative name!) - if (filterInstance) - if ( !filterInstance->matchesFileFilterIncl(fullName.c_str() + prefixLength) || - filterInstance->matchesFileFilterExcl(fullName.c_str() + prefixLength)) - { - statusHandler->requestUiRefresh(); - return TRAVERSING_CONTINUE; - } - FileDescrLine fileDescr; - fileDescr.fullName = fullName; - fileDescr.relativeName = fullName.zsubstr(prefixLength); - fileDescr.lastWriteTimeRaw = details.lastWriteTimeRaw; - fileDescr.fileSize = details.fileSize; - fileDescr.objType = FileDescrLine::TYPE_FILE; - m_output.push_back(fileDescr); - - //assemble status message (performance optimized) = textScanning + wxT("\"") + fullName + wxT("\"") - const unsigned int statusTextMaxLen = 2000; - wxChar statusText[statusTextMaxLen]; - wxChar* position = statusText; - if (textScanning.length() + fullName.length() + 2 < statusTextMaxLen) //leave room for 0 terminating char! +template <bool filterActive> +TraverseCallback::ReturnValue DirCallback<filterActive>::onFile(const DefaultChar* shortName, const Zstring& fullName, const FileInfo& details) +{ + //apply filter before processing (use relative name!) + if (filterActive) + { + if (!baseCallback_->filterInstance->passFileFilter(relNameParentPf_ + shortName)) { - writeText(textScanning.c_str(), textScanning.length(), position); - writeText(wxT("\""), 1, position); - writeText(fullName.c_str(), fullName.length(), position); - writeText(wxT("\""), 1, position); + statusHandler->requestUiRefresh(); + return TRAVERSING_CONTINUE; } - *position = 0; + } - //update UI/commandline status information - statusHandler->updateStatusText(statusText); - //add 1 element to the progress indicator - statusHandler->updateProcessedData(1, 0); //NO performance issue at all - //trigger display refresh - statusHandler->requestUiRefresh(); + output_.addSubFile(FileDescriptor(shortName, details.lastWriteTimeRaw, details.fileSize), relNameParentPf_); - return TRAVERSING_CONTINUE; - } + //assemble status message (performance optimized) = textScanning + wxT("\"") + fullName + wxT("\"") + Zstring statusText = baseCallback_->textScanning; + statusText.reserve(statusText.length() + fullName.length() + 2); + statusText += DefaultChar('\"'); + statusText += fullName; + statusText += DefaultChar('\"'); + + //update UI/commandline status information + statusHandler->updateStatusText(statusText); + //add 1 element to the progress indicator + statusHandler->updateProcessedData(1, 0); //NO performance issue at all + //trigger display refresh + statusHandler->requestUiRefresh(); + return TRAVERSING_CONTINUE; +} - virtual ReturnValDir onDir(const DefaultChar* shortName, const Zstring& fullName) + +template <bool filterActive> +TraverseCallback::ReturnValDir DirCallback<filterActive>::onDir(const DefaultChar* shortName, const Zstring& fullName) +{ + using globalFunctions::FILE_NAME_SEPARATOR; + + Zstring relName = relNameParentPf_; + relName += shortName; + + //apply filter before processing (use relative name!) + if (filterActive) { - //apply filter before processing (use relative name!) - if (filterInstance) + bool subObjMightMatch = true; + if (!baseCallback_->filterInstance->passDirFilter(relName, &subObjMightMatch)) { - if (!filterInstance->matchesDirFilterIncl(fullName.c_str() + prefixLength)) - { - statusHandler->requestUiRefresh(); //if not included: CONTINUE traversing subdirs - return ReturnValDir(ReturnValDir::Continue(), this); - } - else if (filterInstance->matchesDirFilterExcl(fullName.c_str() + prefixLength)) + statusHandler->requestUiRefresh(); + + if (subObjMightMatch) { - statusHandler->requestUiRefresh(); //if excluded: do NOT traverse subdirs - return ReturnValDir::Ignore(); + DirContainer& subDir = output_.addSubDir(DirDescriptor(shortName), relNameParentPf_); + + DirCallback* subDirCallback = new DirCallback(baseCallback_, relName += FILE_NAME_SEPARATOR, subDir, statusHandler); + baseCallback_->callBackBox.push_back(typename BaseDirCallback<filterActive>::CallbackPointer(subDirCallback)); //handle ownership + + //attention: ensure directory filtering is applied to exclude actually filtered directories + return ReturnValDir(ReturnValDir::Continue(), subDirCallback); } + else + return ReturnValDir::Ignore(); //do NOT traverse subdirs } -#ifdef FFS_WIN - if ( fullName.EndsWith(wxT("\\RECYCLER")) || - fullName.EndsWith(wxT("\\System Volume Information"))) - { - statusHandler->requestUiRefresh(); - return ReturnValDir::Ignore(); - } -#endif // FFS_WIN - - FileDescrLine fileDescr; - fileDescr.fullName = fullName; - fileDescr.relativeName = fullName.zsubstr(prefixLength); - fileDescr.lastWriteTimeRaw = 0; //irrelevant for directories - fileDescr.fileSize = 0; //currently used by getBytesToTransfer - fileDescr.objType = FileDescrLine::TYPE_DIRECTORY; - m_output.push_back(fileDescr); - - //assemble status message (performance optimized) = textScanning + wxT("\"") + fullName + wxT("\"") - const unsigned int statusTextMaxLen = 2000; - wxChar statusText[statusTextMaxLen]; - wxChar* position = statusText; - if (textScanning.length() + fullName.length() + 2 < statusTextMaxLen) //leave room for 0 terminating char! - { - writeText(textScanning.c_str(), textScanning.length(), position); - writeText(wxT("\""), 1, position); - writeText(fullName.c_str(), fullName.length(), position); - writeText(wxT("\""), 1, position); - } - *position = 0; + } - //update UI/commandline status information - statusHandler->updateStatusText(statusText); - //add 1 element to the progress indicator - statusHandler->updateProcessedData(1, 0); //NO performance issue at all - //trigger display refresh - statusHandler->requestUiRefresh(); + DirContainer& subDir = output_.addSubDir(DirDescriptor(shortName), relNameParentPf_); - return ReturnValDir(ReturnValDir::Continue(), this); - } + //assemble status message (performance optimized) = textScanning + wxT("\"") + fullName + wxT("\"") + Zstring statusText = baseCallback_->textScanning; + statusText.reserve(statusText.length() + fullName.length() + 2); + statusText += DefaultChar('\"'); + statusText += fullName; + statusText += DefaultChar('\"'); + + //update UI/commandline status information + statusHandler->updateStatusText(statusText); + //add 1 element to the progress indicator + statusHandler->updateProcessedData(1, 0); //NO performance issue at all + //trigger display refresh + statusHandler->requestUiRefresh(); + + DirCallback* subDirCallback = new DirCallback(baseCallback_, relName+=FILE_NAME_SEPARATOR, subDir, statusHandler); + baseCallback_->callBackBox.push_back(typename BaseDirCallback<filterActive>::CallbackPointer(subDirCallback)); //handle ownership + + return ReturnValDir(ReturnValDir::Continue(), subDirCallback); +} - virtual ReturnValue onError(const wxString& errorText) + +template <bool filterActive> +TraverseCallback::ReturnValue DirCallback<filterActive>::onError(const wxString& errorText) +{ + while (true) { - while (true) + switch (statusHandler->reportError(errorText)) { - switch (statusHandler->reportError(errorText)) - { - case ErrorHandler::IGNORE_ERROR: - return TRAVERSING_CONTINUE; - case ErrorHandler::RETRY: - break; //I have to admit "retry" is a bit of a fake here... at least the user has opportunity to abort! - } + case ErrorHandler::IGNORE_ERROR: + return TRAVERSING_CONTINUE; + case ErrorHandler::RETRY: + break; //I have to admit "retry" is a bit of a fake here... at least the user has opportunity to abort! } - - return TRAVERSING_CONTINUE; } -private: - DirectoryDescrType& m_output; - Zstring directory; - int prefixLength; - const Zstring textScanning; - const FilterProcess* const filterInstance; //may be NULL! - StatusHandler* statusHandler; -}; + return TRAVERSING_CONTINUE; //dummy value +} -struct DescrBufferLine +template <bool filterActive> +TraverseCallback::ReturnValDir BaseDirCallback<filterActive>::onDir(const DefaultChar* shortName, const Zstring& fullName) { +//#ifdef FFS_WIN => transparency is more important: just scan every file +// if ( fullName.EndsWith(wxT("\\RECYCLER")) || +// fullName.EndsWith(wxT("\\System Volume Information"))) +// { +// DirCallback<filterActive>::statusHandler->requestUiRefresh(); +// return TraverseCallback::ReturnValDir::Ignore(); +// } +//#endif // FFS_WIN + return DirCallback<filterActive>::onDir(shortName, fullName); +} + + +//------------------------------------------------------------------------------------------ +struct DirBufferKey +{ + DirBufferKey(const Zstring& dirname, + boost::shared_ptr<const FilterProcess>& filterInst) : + directoryName(dirname), + filterInstance(filterInst) {} + Zstring directoryName; - DirectoryDescrType* directoryDesc; + boost::shared_ptr<const FilterProcess> filterInstance; //buffering has to consider filtering! - bool operator < (const DescrBufferLine& b) const + bool operator < (const DirBufferKey& b) const { #ifdef FFS_WIN //Windows does NOT distinguish between upper/lower-case - return (directoryName.CmpNoCase(b.directoryName) < 0); + const int rv = directoryName.CmpNoCase(b.directoryName); #elif defined FFS_LINUX //Linux DOES distinguish between upper/lower-case - return (directoryName.Cmp(b.directoryName) < 0); + const int rv = directoryName.Cmp(b.directoryName); #endif + if (rv != 0) + return rv < 0; + + return filterInstance.get() && b.filterInstance.get() ? + *filterInstance < *b.filterInstance : + filterInstance.get() < b.filterInstance.get(); //at least one of these is NULL } }; -class DirectoryDescrBuffer //buffer multiple scans of the same directories +//------------------------------------------------------------------------------------------ +class CompareProcess::DirectoryBuffer //buffer multiple scans of the same directories { public: - DirectoryDescrBuffer(const bool traverseDirectorySymlinks, const FilterProcess* filter, StatusHandler* statusUpdater) : + DirectoryBuffer(const bool traverseDirectorySymlinks, StatusHandler* statusUpdater) : m_traverseDirectorySymlinks(traverseDirectorySymlinks), - filterInstance(filter), m_statusUpdater(statusUpdater) {} - ~DirectoryDescrBuffer() - { - //clean up - for (std::set<DescrBufferLine>::iterator i = buffer.begin(); i != buffer.end(); ++i) - delete i->directoryDesc; - } + const DirContainer& getDirectoryDescription(const Zstring& directoryPostfixed, bool filterActive, const wxString& includeFilter, const wxString& excludeFilter); - DirectoryDescrType* getDirectoryDescription(const Zstring& directoryFormatted) - { - DescrBufferLine bufferEntry; - bufferEntry.directoryName = directoryFormatted; - - std::set<DescrBufferLine>::iterator entryFound = buffer.find(bufferEntry); - if (entryFound != buffer.end()) - { - //entry found in buffer; return - return entryFound->directoryDesc; - } - else - { - //entry not found; create new one - bufferEntry.directoryDesc = new DirectoryDescrType; - buffer.insert(bufferEntry); //exception safety: insert into buffer right after creation! - - if (FreeFileSync::dirExists(directoryFormatted)) //folder existence already checked in startCompareProcess(): do not treat as error when arriving here! - { - //get all files and folders from directoryFormatted (and subdirectories) - GetAllFilesFull traverser(*bufferEntry.directoryDesc, directoryFormatted, filterInstance, m_statusUpdater); //exceptions may be thrown! - traverseFolder(directoryFormatted, m_traverseDirectorySymlinks, &traverser); - } +private: + typedef boost::shared_ptr<DirContainer> DirBufferValue; //exception safety: avoid memory leak + typedef std::map<DirBufferKey, DirBufferValue> BufferType; - return bufferEntry.directoryDesc; - } - } + static DirBufferKey createKey(const Zstring& directoryPostfixed, bool filterActive, const wxString& includeFilter, const wxString& excludeFilter); + DirContainer& insertIntoBuffer(const DirBufferKey& newKey); -private: - std::set<DescrBufferLine> buffer; + BufferType buffer; const bool m_traverseDirectorySymlinks; - const FilterProcess* const filterInstance; //may be NULL! StatusHandler* m_statusUpdater; }; +//------------------------------------------------------------------------------------------ + + +DirBufferKey CompareProcess::DirectoryBuffer::createKey(const Zstring& directoryPostfixed, bool filterActive, const wxString& includeFilter, const wxString& excludeFilter) +{ + boost::shared_ptr<const FilterProcess> filterInstance( + filterActive && FilterProcess(includeFilter, excludeFilter) != FilterProcess::nullFilter() ? //nullfilter: in: '*', ex '' + new FilterProcess(includeFilter, excludeFilter) : NULL); + + return DirBufferKey(directoryPostfixed, filterInstance); +} -void foldersAreValidForComparison(const std::vector<FolderPair>& folderPairs, StatusHandler* statusUpdater) +DirContainer& CompareProcess::DirectoryBuffer::insertIntoBuffer(const DirBufferKey& newKey) +{ + DirBufferValue baseContainer(new DirContainer); + buffer.insert(std::make_pair(newKey, baseContainer)); + + if (FreeFileSync::dirExists(newKey.directoryName.c_str())) //folder existence already checked in startCompareProcess(): do not treat as error when arriving here! + { + std::auto_ptr<TraverseCallback> traverser(newKey.filterInstance.get() ? + static_cast<TraverseCallback*>(new BaseDirCallback<true>(*baseContainer.get(), newKey.filterInstance.get(), m_statusUpdater)) : + static_cast<TraverseCallback*>(new BaseDirCallback<false>(*baseContainer.get(), NULL, m_statusUpdater))); + + //get all files and folders from directoryPostfixed (and subdirectories) + traverseFolder(newKey.directoryName, m_traverseDirectorySymlinks, traverser.get()); //exceptions may be thrown! + } + return *baseContainer.get(); +} + + +const DirContainer& CompareProcess::DirectoryBuffer::getDirectoryDescription( + const Zstring& directoryPostfixed, + bool filterActive, + const wxString& includeFilter, + const wxString& excludeFilter) +{ + const DirBufferKey searchKey = createKey(directoryPostfixed, filterActive, includeFilter, excludeFilter); + + BufferType::const_iterator entryFound = buffer.find(searchKey); + if (entryFound != buffer.end()) + return *entryFound->second.get(); //entry found in buffer; return + else + return insertIntoBuffer(searchKey); //entry not found; create new one +} + + +//------------------------------------------------------------------------------------------ +void foldersAreValidForComparison(const std::vector<FolderPairCfg>& folderPairsForm, StatusHandler* statusUpdater) { bool checkEmptyDirnameActive = true; //check for empty dirs just once const wxString additionalInfo = _("You can ignore the error to consider not existing directories as empty."); - for (std::vector<FolderPair>::const_iterator i = folderPairs.begin(); i != folderPairs.end(); ++i) + for (std::vector<FolderPairCfg>::const_iterator i = folderPairsForm.begin(); i != folderPairsForm.end(); ++i) { - const Zstring leftFolderName = getFormattedDirectoryName(i->leftDirectory); - const Zstring rightFolderName = getFormattedDirectoryName(i->rightDirectory); - //check if folder name is empty - if (leftFolderName.empty() || rightFolderName.empty()) + if (i->leftDirectory.empty() || i->rightDirectory.empty()) { if (checkEmptyDirnameActive) { @@ -264,11 +353,11 @@ void foldersAreValidForComparison(const std::vector<FolderPair>& folderPairs, St } //check if folders exist - if (!leftFolderName.empty()) - while (!FreeFileSync::dirExists(leftFolderName)) + if (!i->leftDirectory.empty()) + while (!FreeFileSync::dirExists(i->leftDirectory.c_str())) { ErrorHandler::Response rv = statusUpdater->reportError(wxString(_("Directory does not exist:")) + wxT("\n") + - wxT("\"") + leftFolderName + wxT("\"") + wxT("\n\n") + + wxT("\"") + i->leftDirectory + wxT("\"") + wxT("\n\n") + FreeFileSync::getLastErrorFormatted() + wxT(" ") + additionalInfo); if (rv == ErrorHandler::IGNORE_ERROR) break; @@ -278,11 +367,11 @@ void foldersAreValidForComparison(const std::vector<FolderPair>& folderPairs, St assert (false); } - if (!rightFolderName.empty()) - while (!FreeFileSync::dirExists(rightFolderName)) + if (!i->rightDirectory.empty()) + while (!FreeFileSync::dirExists(i->rightDirectory.c_str())) { ErrorHandler::Response rv = statusUpdater->reportError(wxString(_("Directory does not exist:")) + wxT("\n") + - wxT("\"") + rightFolderName + wxT("\"") + wxT("\n\n") + + wxT("\"") + i->rightDirectory + wxT("\"") + wxT("\n\n") + FreeFileSync::getLastErrorFormatted() + wxT(" ") + additionalInfo); if (rv == ErrorHandler::IGNORE_ERROR) break; @@ -295,43 +384,40 @@ void foldersAreValidForComparison(const std::vector<FolderPair>& folderPairs, St } -bool dependencyExists(const std::vector<Zstring>& folders, const Zstring& newFolder, wxString& warningMessage) +bool dependencyExists(const std::set<Zstring>& folders, const Zstring& newFolder, wxString& warningMessage) { - if (!newFolder.empty()) //empty folders names might be accepted by user - { - warningMessage.Clear(); - - for (std::vector<Zstring>::const_iterator i = folders.begin(); i != folders.end(); ++i) - if (!i->empty()) //empty folders names might be accepted by user - if (newFolder.StartsWith(*i) || i->StartsWith(newFolder)) - { - warningMessage = wxString(_("Directories are dependent! Be careful when setting up synchronization rules:")) + wxT("\n") + - wxT("\"") + i->c_str() + wxT("\",\n") + - wxT("\"") + newFolder.c_str() + wxT("\""); - return true; - } - } + for (std::set<Zstring>::const_iterator i = folders.begin(); i != folders.end(); ++i) + if (newFolder.StartsWith(*i) || i->StartsWith(newFolder)) + { + warningMessage = wxString(_("Directories are dependent! Be careful when setting up synchronization rules:")) + wxT("\n") + + wxT("\"") + i->c_str() + wxT("\",\n") + + wxT("\"") + newFolder.c_str() + wxT("\""); + return true; + } return false; } -bool foldersHaveDependencies(const std::vector<FolderPair>& folderPairs, wxString& warningMessage) +bool foldersHaveDependencies(const std::vector<FolderPairCfg>& folderPairsFrom, wxString& warningMessage) { warningMessage.Clear(); - std::vector<Zstring> folders; - for (std::vector<FolderPair>::const_iterator i = folderPairs.begin(); i != folderPairs.end(); ++i) + std::set<Zstring> folders; + for (std::vector<FolderPairCfg>::const_iterator i = folderPairsFrom.begin(); i != folderPairsFrom.end(); ++i) { - const Zstring leftFolderName = getFormattedDirectoryName(i->leftDirectory); - const Zstring rightFolderName = getFormattedDirectoryName(i->rightDirectory); - - if (dependencyExists(folders, leftFolderName, warningMessage)) - return true; - folders.push_back(leftFolderName); + if (!i->leftDirectory.empty()) //empty folders names might be accepted by user + { + if (dependencyExists(folders, i->leftDirectory, warningMessage)) + return true; + folders.insert(i->leftDirectory); + } - if (dependencyExists(folders, rightFolderName, warningMessage)) - return true; - folders.push_back(rightFolderName); + if (!i->rightDirectory.empty()) //empty folders names might be accepted by user + { + if (dependencyExists(folders, i->rightDirectory, warningMessage)) + return true; + folders.insert(i->rightDirectory); + } } return false; @@ -341,8 +427,7 @@ bool foldersHaveDependencies(const std::vector<FolderPair>& folderPairs, wxStrin CompareProcess::CompareProcess(const bool traverseSymLinks, const unsigned int fileTimeTol, const bool ignoreOneHourDiff, - xmlAccess::WarningMessages& warnings, - const FilterProcess* filter, //may be NULL + xmlAccess::OptionalDialogs& warnings, StatusHandler* handler) : fileTimeTolerance(fileTimeTol), ignoreOneHourDifference(ignoreOneHourDiff), @@ -350,13 +435,7 @@ CompareProcess::CompareProcess(const bool traverseSymLinks, statusUpdater(handler), txtComparingContentOfFiles(Zstring(_("Comparing content of files %x")).Replace(wxT("%x"), wxT("\n\"%x\""), false)) { - descriptionBuffer = new DirectoryDescrBuffer(traverseSymLinks, filter, handler); -} - - -CompareProcess::~CompareProcess() -{ - delete descriptionBuffer; + directoryBuffer.reset(new DirectoryBuffer(traverseSymLinks, handler)); } @@ -364,19 +443,16 @@ struct MemoryAllocator { MemoryAllocator() { - buffer1 = new unsigned char[bufferSize]; - buffer2 = new unsigned char[bufferSize]; + buffer = new unsigned char[bufferSize]; } ~MemoryAllocator() { - delete [] buffer1; - delete [] buffer2; + delete [] buffer; } static const unsigned int bufferSize = 512 * 1024; //512 kb seems to be the perfect buffer size - unsigned char* buffer1; - unsigned char* buffer2; + unsigned char* buffer; }; @@ -391,26 +467,27 @@ public: bool filesHaveSameContent(const Zstring& filename1, const Zstring& filename2, CompareCallback* callback) { - static MemoryAllocator memory; + static MemoryAllocator memory1; + static MemoryAllocator memory2; wxFFile file1(filename1.c_str(), wxT("rb")); if (!file1.IsOpened()) - throw FileError(wxString(_("Error reading file:")) + wxT(" \"") + filename1.c_str() + wxT("\"")); + throw FileError(wxString(_("Error opening file:")) + wxT(" \"") + filename1.c_str() + wxT("\"")); wxFFile file2(filename2.c_str(), wxT("rb")); if (!file2.IsOpened()) //NO cleanup necessary for (wxFFile) file1 - throw FileError(wxString(_("Error reading file:")) + wxT(" \"") + filename2.c_str() + wxT("\"")); + throw FileError(wxString(_("Error opening file:")) + wxT(" \"") + filename2.c_str() + wxT("\"")); wxLongLong bytesCompared; do { - const size_t length1 = file1.Read(memory.buffer1, MemoryAllocator::bufferSize); + const size_t length1 = file1.Read(memory1.buffer, MemoryAllocator::bufferSize); if (file1.Error()) throw FileError(wxString(_("Error reading file:")) + wxT(" \"") + filename1.c_str() + wxT("\"")); - const size_t length2 = file2.Read(memory.buffer2, MemoryAllocator::bufferSize); + const size_t length2 = file2.Read(memory2.buffer, MemoryAllocator::bufferSize); if (file2.Error()) throw FileError(wxString(_("Error reading file:")) + wxT(" \"") + filename2.c_str() + wxT("\"")); - if (length1 != length2 || memcmp(memory.buffer1, memory.buffer2, length1) != 0) + if (length1 != length2 || ::memcmp(memory1.buffer, memory2.buffer, length1) != 0) return false; bytesCompared += length1 * 2; @@ -531,10 +608,63 @@ bool filesHaveSameContentUpdating(const Zstring& filename1, const Zstring& filen return cmpAndUpdate.sameContent; }*/ +void formatPair(FolderPairCfg& input) +{ + input.leftDirectory = FreeFileSync::getFormattedDirectoryName(input.leftDirectory); + input.rightDirectory = FreeFileSync::getFormattedDirectoryName(input.rightDirectory); +} + + +struct ToBeRemoved +{ + bool operator()(const DirMapping& dirObj) const + { + return !dirObj.selectedForSynchronization && dirObj.subDirs.size() == 0 && dirObj.subFiles.size() == 0; + } +}; + + +class RemoveFilteredDirs +{ +public: + RemoveFilteredDirs(const wxString& include, const wxString& exclude) : + filterProc(include, exclude) {} + + void operator()(HierarchyObject& hierObj) + { + //process subdirs recursively + std::for_each(hierObj.subDirs.begin(), hierObj.subDirs.end(), *this); + + //filter subdirectories + std::for_each(hierObj.subDirs.begin(), hierObj.subDirs.end(), boost::bind(&RemoveFilteredDirs::filterDir, this, _1)); + + //remove superfluous directories + hierObj.subDirs.erase(std::remove_if(hierObj.subDirs.begin(), hierObj.subDirs.end(), ::ToBeRemoved()), hierObj.subDirs.end()); + } + + void filterDir(FreeFileSync::DirMapping& dirObj) + { + const Zstring relName = dirObj.isEmpty<FreeFileSync::LEFT_SIDE>() ? + dirObj.getRelativeName<FreeFileSync::RIGHT_SIDE>() : + dirObj.getRelativeName<FreeFileSync::LEFT_SIDE>(); + + dirObj.selectedForSynchronization = filterProc.passDirFilter(relName, NULL); //subObjMightMatch is always true in this context! + } + +private: + const FilterProcess filterProc; +}; + + +//filters and removes all excluded directories (but keeps those serving as parent folders) +void filterAndRemoveDirs(BaseDirMapping& baseDir, const wxString& include, const wxString& exclude) +{ + RemoveFilteredDirs(include, exclude)(baseDir); +} + -void CompareProcess::startCompareProcess(const std::vector<FolderPair>& directoryPairs, +void CompareProcess::startCompareProcess(const std::vector<FolderPairCfg>& directoryPairs, const CompareVariant cmpVar, - const SyncConfiguration& config, FolderComparison& output) { #ifndef __WXDEBUG__ @@ -547,11 +677,8 @@ void CompareProcess::startCompareProcess(const std::vector<FolderPair>& director statusUpdater->initNewProcess(-1, 0, StatusHandler::PROCESS_SCANNING); //it's not known how many files will be scanned => -1 objects //format directory pairs: ensure they end with globalFunctions::FILE_NAME_SEPARATOR! - std::vector<FolderPair> directoryPairsFormatted; - for (std::vector<FolderPair>::const_iterator i = directoryPairs.begin(); i != directoryPairs.end(); ++i) - directoryPairsFormatted.push_back( - FolderPair(FreeFileSync::getFormattedDirectoryName(i->leftDirectory), - FreeFileSync::getFormattedDirectoryName(i->rightDirectory))); + std::vector<FolderPairCfg> directoryPairsFormatted = directoryPairs; + std::for_each(directoryPairsFormatted.begin(), directoryPairsFormatted.end(), formatPair); //-------------------some basic checks:------------------------------------------ @@ -581,26 +708,35 @@ void CompareProcess::startCompareProcess(const std::vector<FolderPair>& director break; } - //actually this is the initial determination - FreeFileSync::redetermineSyncDirection(config, output_tmp); + + assert (output_tmp.size() == directoryPairsFormatted.size()); + + for (FolderComparison::iterator j = output_tmp.begin(); j != output_tmp.end(); ++j) + { + const FolderPairCfg& fpCfg = directoryPairsFormatted[j - output_tmp.begin()]; + + //attention: some filtered directories are still in the comparison result! (see include filter handling!) + if (fpCfg.filterIsActive) //let's filter them now... (and remove those that contain excluded elements only) + filterAndRemoveDirs(*j, fpCfg.includeFilter, fpCfg.excludeFilter); + + //set sync-direction initially + FreeFileSync::redetermineSyncDirection(fpCfg.syncConfiguration, *j); + } //only if everything was processed correctly output is written to! //note: output mustn't change during this process to be in sync with GUI grid view!!! output_tmp.swap(output); } - catch (const RuntimeException& theException) - { - statusUpdater->reportFatalError(theException.show().c_str()); - return; //should be obsolete! - } - catch (std::bad_alloc& e) + catch (const std::exception& e) { - statusUpdater->reportFatalError((wxString(_("System out of memory!")) + wxT(" ") + wxString::From8BitData(e.what())).c_str()); + if (dynamic_cast<const std::bad_alloc*>(&e) != NULL) + statusUpdater->reportFatalError(wxString(_("System out of memory!")) + wxT(" ") + wxString::From8BitData(e.what())); + else + statusUpdater->reportFatalError(wxString::From8BitData(e.what())); return; //should be obsolete! } } - //--------------------assemble conflict descriptions--------------------------- //check for very old dates or dates in the future @@ -614,7 +750,7 @@ wxString getConflictInvalidDate(const Zstring& fileNameFull, const wxLongLong& u //check for changed files with same modification date -wxString getConflictSameDateDiffSize(const FileCompareLine& cmpLine) +wxString getConflictSameDateDiffSize(const FileMapping& fileObj) { //some beautification... wxString left = wxString(_("Left")) + wxT(": "); @@ -624,18 +760,18 @@ wxString getConflictSameDateDiffSize(const FileCompareLine& cmpLine) right.Pad(maxPref - right.length(), wxT(' '), true); wxString msg = _("Files %x have the same date but a different size!"); - msg.Replace(wxT("%x"), wxString(wxT("\"")) + cmpLine.fileDescrLeft.relativeName.c_str() + wxT("\"")); + msg.Replace(wxT("%x"), wxString(wxT("\"")) + fileObj.getRelativeName<LEFT_SIDE>() + wxT("\"")); msg += wxT("\n\n"); - msg += left + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(cmpLine.fileDescrLeft.lastWriteTimeRaw, - cmpLine.fileDescrLeft.fullName) + wxT(" \t") + _("Size") + wxT(": ") + cmpLine.fileDescrLeft.fileSize.ToString() + wxT("\n"); - msg += right + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(cmpLine.fileDescrRight.lastWriteTimeRaw, - cmpLine.fileDescrRight.fullName) + wxT(" \t") + _("Size") + wxT(": ") + cmpLine.fileDescrRight.fileSize.ToString(); + msg += left + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(fileObj.getLastWriteTime<LEFT_SIDE>(), + fileObj.getFullName<LEFT_SIDE>()) + wxT(" \t") + _("Size") + wxT(": ") + fileObj.getFileSize<LEFT_SIDE>().ToString() + wxT("\n"); + msg += right + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(fileObj.getLastWriteTime<RIGHT_SIDE>(), + fileObj.getFullName<RIGHT_SIDE>()) + wxT(" \t") + _("Size") + wxT(": ") + fileObj.getFileSize<RIGHT_SIDE>().ToString(); return wxString(_("Conflict detected:")) + wxT("\n") + msg; } //check for files that have a difference in file modification date below 1 hour when DST check is active -wxString getConflictChangeWithinHour(const FileCompareLine& cmpLine) +wxString getConflictChangeWithinHour(const FileMapping& fileObj) { //some beautification... wxString left = wxString(_("Left")) + wxT(": "); @@ -644,19 +780,16 @@ wxString getConflictChangeWithinHour(const FileCompareLine& cmpLine) left.Pad(maxPref - left.length(), wxT(' '), true); right.Pad(maxPref - right.length(), wxT(' '), true); - wxString msg = _("Files %x have a file time difference of less than 1 hour! It's not safe to decide which one is newer due to Daylight Saving Time issues."); - msg.Replace(wxT("%x"), wxString(wxT("\"")) + cmpLine.fileDescrLeft.relativeName.c_str() + wxT("\"")); + wxString msg = _("Files %x have a file time difference of less than 1 hour!\n\nIt's not safe to decide which one is newer due to Daylight Saving Time issues."); + msg += wxString(wxT("\n")) + _("(Note that only FAT/FAT32 drives are affected by this problem!\nIn all other cases you can disable the setting \"ignore 1-hour difference\".)"); + msg.Replace(wxT("%x"), wxString(wxT("\"")) + fileObj.getRelativeName<LEFT_SIDE>() + wxT("\"")); msg += wxT("\n\n"); - msg += left + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(cmpLine.fileDescrLeft.lastWriteTimeRaw, cmpLine.fileDescrLeft.fullName) + wxT("\n"); - msg += right + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(cmpLine.fileDescrRight.lastWriteTimeRaw, cmpLine.fileDescrRight.fullName); + msg += left + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(fileObj.getLastWriteTime<LEFT_SIDE>(), fileObj.getFullName<LEFT_SIDE>()) + wxT("\n"); + msg += right + wxT("\t") + _("Date") + wxT(": ") + utcTimeToLocalString(fileObj.getLastWriteTime<RIGHT_SIDE>(), fileObj.getFullName<RIGHT_SIDE>()); return wxString(_("Conflict detected:")) + wxT("\n") + msg; } -//----------------------------------------------------------------------------- - - -const CompareFilesResult FILE_UNDEFINED = CompareFilesResult(42); - +//----------------------------------------------------------------------------- inline bool sameFileTime(const wxLongLong& a, const wxLongLong& b, const unsigned tolerance) { @@ -667,385 +800,344 @@ bool sameFileTime(const wxLongLong& a, const wxLongLong& b, const unsigned toler } -void CompareProcess::compareByTimeSize(const std::vector<FolderPair>& directoryPairsFormatted, FolderComparison& output) +void CompareProcess::compareByTimeSize(const std::vector<FolderPairCfg>& directoryPairsFormatted, FolderComparison& output) { //process one folder pair after each other - for (std::vector<FolderPair>::const_iterator pair = directoryPairsFormatted.begin(); pair != directoryPairsFormatted.end(); ++pair) + for (std::vector<FolderPairCfg>::const_iterator pair = directoryPairsFormatted.begin(); pair != directoryPairsFormatted.end(); ++pair) { - FolderCompareLine newEntry; - newEntry.syncPair = *pair; + BaseDirMapping newEntry(pair->leftDirectory, pair->rightDirectory); output.push_back(newEntry); //attention: push_back() copies by value!!! performance: append BEFORE writing values into fileCmp! - FileComparison& fileCmp = output.back().fileCmp; + //do basis scan and retrieve files existing on both sides + std::vector<FileMapping*> compareCandidates; + performBaseComparison(*pair, output.back(), compareCandidates); - //do basis scan: only result lines of type FILE_UNDEFINED (files that exist on both sides) need to be determined after this call - this->performBaseComparison(*pair, fileCmp); + //PERF_START; //categorize files that exist on both sides - for (FileComparison::iterator i = fileCmp.begin(); i != fileCmp.end(); ++i) - if (i->cmpResult == FILE_UNDEFINED) + for (std::vector<FileMapping*>::iterator i = compareCandidates.begin(); i != compareCandidates.end(); ++i) + { + FileMapping* const line = *i; + if (line->getLastWriteTime<LEFT_SIDE>() != line->getLastWriteTime<RIGHT_SIDE>()) { - if (i->fileDescrLeft.lastWriteTimeRaw != i->fileDescrRight.lastWriteTimeRaw) + //number of seconds since Jan 1st 1970 + 1 year (needn't be too precise) + static const long oneYearFromNow = wxGetUTCTime() + 365 * 24 * 3600; + + //check for erroneous dates (but only if dates are not (EXACTLY) the same) + if ( line->getLastWriteTime<LEFT_SIDE>() < 0 || //earlier than Jan 1st 1970 + line->getLastWriteTime<RIGHT_SIDE>() < 0 || //earlier than Jan 1st 1970 + line->getLastWriteTime<LEFT_SIDE>() > oneYearFromNow || //dated more than one year in future + line->getLastWriteTime<RIGHT_SIDE>() > oneYearFromNow) //dated more than one year in future + { + line->cmpResult = FILE_CONFLICT; + if (line->getLastWriteTime<LEFT_SIDE>() < 0 || line->getLastWriteTime<LEFT_SIDE>() > oneYearFromNow) + line->conflictDescription = getConflictInvalidDate(line->getFullName<LEFT_SIDE>(), line->getLastWriteTime<LEFT_SIDE>()); + else + line->conflictDescription = getConflictInvalidDate(line->getFullName<RIGHT_SIDE>(), line->getLastWriteTime<RIGHT_SIDE>()); + } + else //from this block on all dates are at least "valid" { - //number of seconds since Jan 1st 1970 + 1 year (needn't be too precise) - static const long oneYearFromNow = wxGetUTCTime() + 365 * 24 * 3600; - - //check for erroneous dates (but only if dates are not (EXACTLY) the same) - if ( i->fileDescrLeft.lastWriteTimeRaw < 0 || //earlier than Jan 1st 1970 - i->fileDescrRight.lastWriteTimeRaw < 0 || //earlier than Jan 1st 1970 - i->fileDescrLeft.lastWriteTimeRaw > oneYearFromNow || //dated more than one year in future - i->fileDescrRight.lastWriteTimeRaw > oneYearFromNow) //dated more than one year in future + //last write time may differ by up to 2 seconds (NTFS vs FAT32) + if (sameFileTime(line->getLastWriteTime<LEFT_SIDE>(), line->getLastWriteTime<RIGHT_SIDE>(), fileTimeTolerance)) { - i->cmpResult = FILE_CONFLICT; - if (i->fileDescrLeft.lastWriteTimeRaw < 0 || i->fileDescrLeft.lastWriteTimeRaw > oneYearFromNow) - i->conflictDescription = OptionalString(getConflictInvalidDate(i->fileDescrLeft.fullName, i->fileDescrLeft.lastWriteTimeRaw)); + if (line->getFileSize<LEFT_SIDE>() == line->getFileSize<RIGHT_SIDE>()) + line->cmpResult = FILE_EQUAL; else - i->conflictDescription = OptionalString(getConflictInvalidDate(i->fileDescrRight.fullName, i->fileDescrRight.lastWriteTimeRaw)); + { + line->cmpResult = FILE_CONFLICT; //same date, different filesize + line->conflictDescription = getConflictSameDateDiffSize(*line); + } } - else //from this block on all dates are at least "valid" + else { - //last write time may differ by up to 2 seconds (NTFS vs FAT32) - if (sameFileTime(i->fileDescrLeft.lastWriteTimeRaw, i->fileDescrRight.lastWriteTimeRaw, fileTimeTolerance)) + //finally: DST +/- 1-hour check: test if time diff is exactly +/- 1-hour (respecting 2 second FAT precision) + if (ignoreOneHourDifference && sameFileTime(line->getLastWriteTime<LEFT_SIDE>(), line->getLastWriteTime<RIGHT_SIDE>(), 3600 + 2)) { - if (i->fileDescrLeft.fileSize == i->fileDescrRight.fileSize) - i->cmpResult = FILE_EQUAL; - else + //date diff < 1 hour is a conflict: it's not safe to determine which file is newer + if (sameFileTime(line->getLastWriteTime<LEFT_SIDE>(), line->getLastWriteTime<RIGHT_SIDE>(), 3600 - 2 - 1)) { - i->cmpResult = FILE_CONFLICT; //same date, different filesize - i->conflictDescription = OptionalString(getConflictSameDateDiffSize(*i)); + line->cmpResult = FILE_CONFLICT; + line->conflictDescription = getConflictChangeWithinHour(*line); } - } - else - { - //finally: DST +/- 1-hour check: test if time diff is exactly +/- 1-hour (respecting 2 second FAT precision) - if (ignoreOneHourDifference && sameFileTime(i->fileDescrLeft.lastWriteTimeRaw, i->fileDescrRight.lastWriteTimeRaw, 3600 + 2)) + else //exact +/- 1-hour detected: treat as equal { - //date diff < 1 hour is a conflict: it's not safe to determine which file is newer - if (sameFileTime(i->fileDescrLeft.lastWriteTimeRaw, i->fileDescrRight.lastWriteTimeRaw, 3600 - 2 - 1)) - { - i->cmpResult = FILE_CONFLICT; - i->conflictDescription = OptionalString(getConflictChangeWithinHour(*i)); - } - else //exact +/- 1-hour detected: treat as equal + if (line->getFileSize<LEFT_SIDE>() == line->getFileSize<RIGHT_SIDE>()) + line->cmpResult = FILE_EQUAL; + else { - if (i->fileDescrLeft.fileSize == i->fileDescrRight.fileSize) - i->cmpResult = FILE_EQUAL; - else - { - i->cmpResult = FILE_CONFLICT; //same date, different filesize - i->conflictDescription = OptionalString(getConflictSameDateDiffSize(*i)); - } + line->cmpResult = FILE_CONFLICT; //same date, different filesize + line->conflictDescription = getConflictSameDateDiffSize(*line); } } + } + else + { + if (line->getLastWriteTime<LEFT_SIDE>() < line->getLastWriteTime<RIGHT_SIDE>()) + line->cmpResult = FILE_RIGHT_NEWER; else - { - if (i->fileDescrLeft.lastWriteTimeRaw < i->fileDescrRight.lastWriteTimeRaw) - i->cmpResult = FILE_RIGHT_NEWER; - else - i->cmpResult = FILE_LEFT_NEWER; - } + line->cmpResult = FILE_LEFT_NEWER; } } } - else //same write time + } + else //same write time + { + if (line->getFileSize<LEFT_SIDE>() == line->getFileSize<RIGHT_SIDE>()) + line->cmpResult = FILE_EQUAL; + else { - if (i->fileDescrLeft.fileSize == i->fileDescrRight.fileSize) - i->cmpResult = FILE_EQUAL; - else - { - i->cmpResult = FILE_CONFLICT; //same date, different filesize - i->conflictDescription = OptionalString(getConflictSameDateDiffSize(*i)); - } + line->cmpResult = FILE_CONFLICT; //same date, different filesize + line->conflictDescription = getConflictSameDateDiffSize(*line); } } + } } } -class RemoveAtExit //this class ensures, that the result of the method below is ALWAYS written on exit, even if exceptions are thrown! -{ -public: - RemoveAtExit(FileComparison& fileCmp) : - gridToWrite(fileCmp) {} - - ~RemoveAtExit() - { - globalFunctions::removeRowsFromVector(rowsToDelete, gridToWrite); - } - - void markRow(int nr) - { - rowsToDelete.insert(nr); - } - -private: - FileComparison& gridToWrite; - std::set<int> rowsToDelete; -}; - - -void getBytesToCompare(const FolderComparison& grid, const FolderCompRef& rowsToCompare, int& objectsTotal, wxULongLong& dataTotal) +wxULongLong getBytesToCompare(const std::vector<FileMapping*>& rowsToCompare) { - objectsTotal = 0; - dataTotal = 0; - - for (FolderComparison::const_iterator j = grid.begin(); j != grid.end(); ++j) - { - const FileComparison& fileCmp = j->fileCmp; + wxULongLong dataTotal; - const std::set<int>& index = rowsToCompare[j - grid.begin()]; - for (std::set<int>::const_iterator i = index.begin(); i != index.end(); ++i) - { - const FileCompareLine& line = fileCmp[*i]; - dataTotal += line.fileDescrLeft.fileSize; - dataTotal += line.fileDescrRight.fileSize; - } + for (std::vector<FileMapping*>::const_iterator j = rowsToCompare.begin(); j != rowsToCompare.end(); ++j) + dataTotal += (*j)->getFileSize<LEFT_SIDE>(); //left and right filesizes should be the same - objectsTotal += index.size() * 2; - } + return dataTotal * 2; } -void CompareProcess::compareByContent(const std::vector<FolderPair>& directoryPairsFormatted, FolderComparison& output) +void CompareProcess::compareByContent(const std::vector<FolderPairCfg>& directoryPairsFormatted, FolderComparison& output) { //PERF_START; + std::vector<FileMapping*> compareCandidates; //process one folder pair after each other - for (std::vector<FolderPair>::const_iterator pair = directoryPairsFormatted.begin(); pair != directoryPairsFormatted.end(); ++pair) + for (std::vector<FolderPairCfg>::const_iterator pair = directoryPairsFormatted.begin(); pair != directoryPairsFormatted.end(); ++pair) { - FolderCompareLine newEntry; - newEntry.syncPair = *pair; + BaseDirMapping newEntry(pair->leftDirectory, pair->rightDirectory); output.push_back(newEntry); //attention: push_back() copies by value!!! performance: append BEFORE writing values into fileCmp! - FileComparison& fileCmp = output.back().fileCmp; - - //do basis scan: only result lines of type FILE_UNDEFINED (files that exist on both sides) need to be determined after this call - this->performBaseComparison(*pair, fileCmp); + //do basis scan and retrieve candidates for binary comparison (files existing on both sides) + performBaseComparison(*pair, output.back(), compareCandidates); } //finish categorization... + std::vector<FileMapping*> filesToCompareBytewise; - FolderCompRef rowsToCompareBytewise; //content comparison of file content happens AFTER finding corresponding files + //content comparison of file content happens AFTER finding corresponding files //in order to separate into two processes (scanning and comparing) - for (FolderComparison::iterator j = output.begin(); j != output.end(); ++j) + for (std::vector<FileMapping*>::iterator i = compareCandidates.begin(); i != compareCandidates.end(); ++i) { - FileComparison& fileCmp = j->fileCmp; + //pre-check: files have different content if they have a different filesize + if ((*i)->getFileSize<LEFT_SIDE>() != (*i)->getFileSize<RIGHT_SIDE>()) + (*i)->cmpResult = FILE_DIFFERENT; + else + filesToCompareBytewise.push_back(*i); + } - std::set<int> newEntry; - for (FileComparison::iterator i = fileCmp.begin(); i != fileCmp.end(); ++i) - { - if (i->cmpResult == FILE_UNDEFINED) - { //pre-check: files have different content if they have a different filesize - if (i->fileDescrLeft.fileSize != i->fileDescrRight.fileSize) - i->cmpResult = FILE_DIFFERENT; - else - newEntry.insert(i - fileCmp.begin()); - } - } - rowsToCompareBytewise.push_back(newEntry); - } - int objectsTotal = 0; - wxULongLong dataTotal; - getBytesToCompare(output, rowsToCompareBytewise, objectsTotal, dataTotal); + const int objectsTotal = filesToCompareBytewise.size() * 2; + const wxULongLong bytesTotal = getBytesToCompare(filesToCompareBytewise); statusUpdater->initNewProcess(objectsTotal, - globalFunctions::convertToSigned(dataTotal), + globalFunctions::convertToSigned(bytesTotal), StatusHandler::PROCESS_COMPARING_CONTENT); //compare files (that have same size) bytewise... - for (FolderComparison::iterator j = output.begin(); j != output.end(); ++j) + for (std::vector<FileMapping*>::const_iterator j = filesToCompareBytewise.begin(); j != filesToCompareBytewise.end(); ++j) { - FileComparison& fileCmp = j->fileCmp; + FileMapping* const gridline = *j; - //mark erroneous rows for deletion from output - RemoveAtExit removeRowsAtExit(fileCmp); //note: running at individual folder pair level! + Zstring statusText = txtComparingContentOfFiles; + statusText.Replace(wxT("%x"), gridline->getRelativeName<LEFT_SIDE>(), false); + statusUpdater->updateStatusText(statusText); - const std::set<int>& index = rowsToCompareBytewise[j - output.begin()]; - for (std::set<int>::const_iterator i = index.begin(); i != index.end(); ++i) + //check files that exist in left and right model but have different content + while (true) { - FileCompareLine& gridline = fileCmp[*i]; - - Zstring statusText = txtComparingContentOfFiles; - statusText.Replace(wxT("%x"), gridline.fileDescrLeft.relativeName.c_str(), false); - statusUpdater->updateStatusText(statusText); + //trigger display refresh + statusUpdater->requestUiRefresh(); - //check files that exist in left and right model but have different content - while (true) + try { - //trigger display refresh - statusUpdater->requestUiRefresh(); + if (filesHaveSameContentUpdating(gridline->getFullName<LEFT_SIDE>(), + gridline->getFullName<RIGHT_SIDE>(), + gridline->getFileSize<LEFT_SIDE>() * 2, + statusUpdater)) + gridline->cmpResult = FILE_EQUAL; + else + gridline->cmpResult = FILE_DIFFERENT; - try + statusUpdater->updateProcessedData(2, 0); //processed data is communicated in subfunctions! + break; + } + catch (FileError& error) + { + ErrorHandler::Response rv = statusUpdater->reportError(error.show()); + if (rv == ErrorHandler::IGNORE_ERROR) { - if (filesHaveSameContentUpdating(gridline.fileDescrLeft.fullName, - gridline.fileDescrRight.fullName, - gridline.fileDescrLeft.fileSize * 2, - statusUpdater)) - gridline.cmpResult = FILE_EQUAL; - else - gridline.cmpResult = FILE_DIFFERENT; - - statusUpdater->updateProcessedData(2, 0); //processed data is communicated in subfunctions! + gridline->cmpResult = FILE_CONFLICT; //same date, different filesize + gridline->conflictDescription = wxString(_("Conflict detected:")) + wxT("\n") + _("Comparing files by content failed."); break; } - catch (FileError& error) - { - ErrorHandler::Response rv = statusUpdater->reportError(error.show()); - if (rv == ErrorHandler::IGNORE_ERROR) - { - removeRowsAtExit.markRow(*i); - break; - } - else if (rv == ErrorHandler::RETRY) - ; //continue with loop - else - assert (false); - } + + else if (rv == ErrorHandler::RETRY) + ; //continue with loop + else + assert (false); } } } } -class ThreadSorting : public wxThread +class MergeSides { public: - ThreadSorting(DirectoryDescrType* directory) : - wxThread(wxTHREAD_JOINABLE), - m_directory(directory) + MergeSides(const Zstring& baseDirLeftPf, + const Zstring& baseDirRightPf, + std::vector<FileMapping*>& appendUndefinedOut) : + baseDirLeft(baseDirLeftPf), + baseDirRight(baseDirRightPf), + appendUndefined(appendUndefinedOut) {} + + void execute(const DirContainer& leftSide, const DirContainer& rightSide, HierarchyObject& output) { - if (Create() != wxTHREAD_NO_ERROR) - throw RuntimeException(wxString(wxT("Error creating thread for sorting!"))); - } + //ATTENTION: HierarchyObject::retrieveById() can only work correctly if the following conditions are fulfilled: + //1. on each level, files are added first, then directories (=> file id < dir id) + //2. when a directory is added, all subdirectories must be added immediately (recursion) before the next dir on this level is added + //3. entries may be deleted but NEVER new ones inserted!!! + //=> this allows for a quasi-binary search by id! - ~ThreadSorting() {} + //reserve() fulfills two task here: 1. massive performance improvement! 2. ensure references in appendUndefined remain valid! + output.subFiles.reserve(leftSide.getSubFiles().size() + rightSide.getSubFiles().size()); //assume worst case! + output.subDirs.reserve( leftSide.getSubDirs().size() + rightSide.getSubDirs().size()); // + for (DirContainer::SubFileList::const_iterator i = leftSide.getSubFiles().begin(); i != leftSide.getSubFiles().end(); ++i) + { + DirContainer::SubFileList::const_iterator j = rightSide.getSubFiles().find(*i); + + //find files that exist on left but not on right + if (j == rightSide.getSubFiles().end()) + output.addSubFile(i->getData(), FILE_LEFT_SIDE_ONLY, FileMapping::nullData(), + RelNamesBuffered(baseDirLeft, //base sync dir postfixed + baseDirRight, + i->getParentRelNamePf())); //relative parent name postfixed + //find files that exist on left and right + else + { + appendUndefined.push_back( + &output.addSubFile(i->getData(), FILE_EQUAL, j->getData(), //FILE_EQUAL is just a dummy-value here + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf()))); + } + } - ExitCode Entry() - { - std::sort(m_directory->begin(), m_directory->end()); - return 0; - } + //find files that exist on right but not on left + for (DirContainer::SubFileList::const_iterator j = rightSide.getSubFiles().begin(); j != rightSide.getSubFiles().end(); ++j) + { + if (leftSide.getSubFiles().find(*j) == leftSide.getSubFiles().end()) + output.addSubFile(FileMapping::nullData(), FILE_RIGHT_SIDE_ONLY, j->getData(), + RelNamesBuffered(baseDirLeft, baseDirRight, j->getParentRelNamePf())); + } -private: - DirectoryDescrType* m_directory; -}; +//----------------------------------------------------------------------------------------------- + for (DirContainer::SubDirList::const_iterator i = leftSide.getSubDirs().begin(); i != leftSide.getSubDirs().end(); ++i) + { + DirContainer::SubDirList::const_iterator j = rightSide.getSubDirs().find(*i); -void CompareProcess::performBaseComparison(const FolderPair& pair, FileComparison& output) -{ - //PERF_START; - //retrieve sets of files (with description data) - DirectoryDescrType* directoryLeft = descriptionBuffer->getDirectoryDescription(pair.leftDirectory); - DirectoryDescrType* directoryRight = descriptionBuffer->getDirectoryDescription(pair.rightDirectory); + //find directories that exist on left but not on right + if (j == rightSide.getSubDirs().end()) + { + DirMapping& newDirMap = output.addSubDir(i->getData(), DIR_LEFT_SIDE_ONLY, DirMapping::nullData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())); + fillOneSide<true>(*i, newDirMap); //recurse into subdirectories + } + else //directories that exist on both sides + { + DirMapping& newDirMap = output.addSubDir(i->getData(), DIR_EQUAL, j->getData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())); + execute(*i, *j, newDirMap); //recurse into subdirectories + } + } - statusUpdater->updateStatusText(_("Generating file list...")); - statusUpdater->forceUiRefresh(); //keep total number of scanned files up to date - //PERF_STOP; + //find directories that exist on right but not on left + for (DirContainer::SubDirList::const_iterator j = rightSide.getSubDirs().begin(); j != rightSide.getSubDirs().end(); ++j) + { + if (leftSide.getSubDirs().find(*j) == leftSide.getSubDirs().end()) + { + DirMapping& newDirMap = output.addSubDir(DirMapping::nullData(), DIR_RIGHT_SIDE_ONLY, j->getData(), + RelNamesBuffered(baseDirLeft, baseDirRight, j->getParentRelNamePf())); + fillOneSide<false>(*j, newDirMap); //recurse into subdirectories + } + } + } - //we use binary search when comparing the directory structures: so sort() first - if (wxThread::GetCPUCount() >= 2) //do it the multithreaded way: +private: + template <bool leftSide> + void fillOneSide(const DirContainer& dirCont, HierarchyObject& output) { - //no synchronization (multithreading) needed here: directoryLeft and directoryRight are disjunct - //reference counting Zstring also shouldn't be an issue, as no strings are deleted during std::sort() - std::auto_ptr<ThreadSorting> sortLeft(new ThreadSorting(directoryLeft)); - std::auto_ptr<ThreadSorting> sortRight(new ThreadSorting(directoryRight)); + //reserve() fulfills two task here: 1. massive performance improvement! 2. ensure references in appendUndefined remain valid! + output.subFiles.reserve(dirCont.getSubFiles().size()); + output.subDirs.reserve( dirCont.getSubDirs(). size()); - if (sortLeft->Run() != wxTHREAD_NO_ERROR) - throw RuntimeException(wxString(wxT("Error starting thread for sorting!"))); + for (DirContainer::SubFileList::const_iterator i = dirCont.getSubFiles().begin(); i != dirCont.getSubFiles().end(); ++i) + { + if (leftSide) + output.addSubFile(i->getData(), FILE_LEFT_SIDE_ONLY, FileMapping::nullData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())); + else + output.addSubFile(FileMapping::nullData(), FILE_RIGHT_SIDE_ONLY, i->getData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())); + } - if (directoryLeft != directoryRight) //attention: might point to the same vector because of buffer! + for (DirContainer::SubDirList::const_iterator i = dirCont.getSubDirs().begin(); i != dirCont.getSubDirs().end(); ++i) { - if (sortRight->Run() != wxTHREAD_NO_ERROR) - throw RuntimeException(wxString(wxT("Error starting thread for sorting!"))); + DirMapping& newDirMap = leftSide ? + output.addSubDir(i->getData(), DIR_LEFT_SIDE_ONLY, DirMapping::nullData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())) : + output.addSubDir(DirMapping::nullData(), DIR_RIGHT_SIDE_ONLY, i->getData(), + RelNamesBuffered(baseDirLeft, baseDirRight, i->getParentRelNamePf())); - if (sortRight->Wait() != 0) - throw RuntimeException(wxString(wxT("Error waiting for thread (sorting)!"))); + fillOneSide<leftSide>(*i, newDirMap); //recurse into subdirectories } - - if (sortLeft->Wait() != 0) - throw RuntimeException(wxString(wxT("Error waiting for thread (sorting)!"))); } - else //single threaded - { - std::sort(directoryLeft->begin(), directoryLeft->end()); - if (directoryLeft != directoryRight) //attention: might point to the same vector because of buffer! - std::sort(directoryRight->begin(), directoryRight->end()); - } - //PERF_STOP; - //reserve some space to avoid too many vector reallocations: doesn't make much sense for multiple folder pairs, but doesn't hurt either - output.reserve(output.size() + unsigned(std::max(directoryLeft->size(), directoryRight->size()) * 1.2)); + const Zstring& baseDirLeft; + const Zstring& baseDirRight; + std::vector<FileMapping*>& appendUndefined; +}; - //begin base comparison - FileCompareLine newline(FILE_UNDEFINED, SYNC_DIR_NONE, true); - for (DirectoryDescrType::const_iterator i = directoryLeft->begin(); i != directoryLeft->end(); ++i) - { - //find files/folders that exist in left file tree but not in right one - DirectoryDescrType::const_iterator j = custom_binary_search(directoryRight->begin(), directoryRight->end(), *i); - if (j == directoryRight->end()) - { - newline.fileDescrLeft = *i; - newline.fileDescrRight = FileDescrLine(); - newline.cmpResult = FILE_LEFT_SIDE_ONLY; - output.push_back(newline); - } - //find files/folders that exist on left and right side - else - { - const FileDescrLine::ObjectType typeLeft = i->objType; - const FileDescrLine::ObjectType typeRight = j->objType; - //files... - if (typeLeft == FileDescrLine::TYPE_FILE && typeRight == FileDescrLine::TYPE_FILE) - { - newline.fileDescrLeft = *i; - newline.fileDescrRight = *j; - newline.cmpResult = FILE_UNDEFINED; //not yet determined! - output.push_back(newline); - } - //directories... - else if (typeLeft == FileDescrLine::TYPE_DIRECTORY && typeRight == FileDescrLine::TYPE_DIRECTORY) - { - newline.fileDescrLeft = *i; - newline.fileDescrRight = *j; - newline.cmpResult = FILE_EQUAL; - output.push_back(newline); - } - //if we have a nameclash between a file and a directory: split into two separate rows - else - { - assert (typeLeft != typeRight); +void CompareProcess::performBaseComparison(const FolderPairCfg& pair, BaseDirMapping& output, std::vector<FileMapping*>& appendUndefined) +{ + assert(output.subDirs.empty()); + assert(output.subFiles.empty()); - newline.fileDescrLeft = *i; - newline.fileDescrRight = FileDescrLine(); - newline.cmpResult = FILE_LEFT_SIDE_ONLY; - output.push_back(newline); + //PERF_START; - newline.fileDescrLeft = FileDescrLine(); - newline.fileDescrRight = *j; - newline.cmpResult = FILE_RIGHT_SIDE_ONLY; - output.push_back(newline); - } - } - } + //scan directories + const DirContainer& directoryLeft = directoryBuffer->getDirectoryDescription(pair.leftDirectory, + pair.filterIsActive, + pair.includeFilter, + pair.excludeFilter); + const DirContainer& directoryRight = directoryBuffer->getDirectoryDescription(pair.rightDirectory, + pair.filterIsActive, + pair.includeFilter, + pair.excludeFilter); - for (DirectoryDescrType::const_iterator j = directoryRight->begin(); j != directoryRight->end(); ++j) - { - //find files/folders that exist in right file model but not in left model - if (custom_binary_search(directoryLeft->begin(), directoryLeft->end(), *j) == directoryLeft->end()) - { - newline.fileDescrLeft = FileDescrLine(); - newline.fileDescrRight = *j; - newline.cmpResult = FILE_RIGHT_SIDE_ONLY; - output.push_back(newline); - } - } - //PERF_STOP + statusUpdater->updateStatusText(_("Generating file list...")); + statusUpdater->forceUiRefresh(); //keep total number of scanned files up to date + + //PERF_STOP; + + MergeSides(pair.leftDirectory, + pair.rightDirectory, + appendUndefined).execute(directoryLeft, directoryRight, output); + //PERF_STOP; } |