From 420fb6c9b3427f65cfe24411944ee46b58cfcfb4 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Fri, 18 Apr 2014 16:57:03 +0200 Subject: 1.17 --- comparison.cpp | 131 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 39 deletions(-) (limited to 'comparison.cpp') diff --git a/comparison.cpp b/comparison.cpp index f90b47de..43191d38 100644 --- a/comparison.cpp +++ b/comparison.cpp @@ -8,6 +8,7 @@ #include "library/multithreading.h" #include "algorithm.h" #include +#include #ifdef FFS_WIN #include //includes "windows.h" @@ -37,12 +38,12 @@ public: } - wxDirTraverseResult GetAllFilesFull::OnFile(const Zstring& fullFileName, const FileInfo& details) //virtual impl. + wxDirTraverseResult OnFile(const Zstring& fullFileName, const FileInfo& details) //virtual impl. { FileDescrLine fileDescr; fileDescr.fullName = fullFileName; fileDescr.directory = directory; - fileDescr.relativeName = fullFileName.substr(prefixLength); + fileDescr.relativeName = fullFileName.zsubstr(prefixLength); fileDescr.lastWriteTimeRaw = details.lastWriteTimeRaw; fileDescr.fileSize = details.fileSize; fileDescr.objType = FileDescrLine::TYPE_FILE; @@ -72,7 +73,7 @@ public: } - wxDirTraverseResult GetAllFilesFull::OnDir(const Zstring& fullDirName) //virtual impl. + wxDirTraverseResult OnDir(const Zstring& fullDirName) //virtual impl. { #ifdef FFS_WIN if ( fullDirName.EndsWith(wxT("\\RECYCLER")) || @@ -83,7 +84,7 @@ public: FileDescrLine fileDescr; fileDescr.fullName = fullDirName; fileDescr.directory = directory; - fileDescr.relativeName = fullDirName.substr(prefixLength); + fileDescr.relativeName = fullDirName.zsubstr(prefixLength); fileDescr.lastWriteTimeRaw = 0; //irrelevant for directories fileDescr.fileSize = wxULongLong(0); //currently used by getBytesToTransfer fileDescr.objType = FileDescrLine::TYPE_DIRECTORY; @@ -113,7 +114,7 @@ public: } - wxDirTraverseResult GetAllFilesFull::OnError(const Zstring& errorText) //virtual impl. + wxDirTraverseResult OnError(const Zstring& errorText) //virtual impl. { while (true) { @@ -160,8 +161,8 @@ struct DescrBufferLine class FreeFileSync::DirectoryDescrBuffer //buffer multiple scans of the same directories { public: - DirectoryDescrBuffer(const bool traverseSymbolicLinks, StatusHandler* statusUpdater) : - m_traverseSymbolicLinks(traverseSymbolicLinks), + DirectoryDescrBuffer(const bool traverseDirectorySymlinks, StatusHandler* statusUpdater) : + m_traverseDirectorySymlinks(traverseDirectorySymlinks), m_statusUpdater(statusUpdater) {} ~DirectoryDescrBuffer() @@ -188,11 +189,9 @@ public: bufferEntry.directoryDesc = new DirectoryDescrType; buffer.insert(bufferEntry); //exception safety: insert into buffer right after creation! - bufferEntry.directoryDesc->reserve(400000); //reserve space for up to 400000 files to avoid too many vector reallocations - //get all files and folders from directoryFormatted (and subdirectories) GetAllFilesFull traverser(*bufferEntry.directoryDesc, directoryFormatted, m_statusUpdater); //exceptions may be thrown! - traverseInDetail(directoryFormatted, m_traverseSymbolicLinks, &traverser); + traverseInDetail(directoryFormatted, m_traverseDirectorySymlinks, &traverser); return bufferEntry.directoryDesc; } @@ -201,7 +200,7 @@ public: private: std::set buffer; - const bool m_traverseSymbolicLinks; + const bool m_traverseDirectorySymlinks; StatusHandler* m_statusUpdater; }; @@ -245,7 +244,7 @@ bool dependencyExists(const std::vector& folders, const Zstring& newFol for (std::vector::const_iterator i = folders.begin(); i != folders.end(); ++i) if (newFolder.StartsWith(*i) || i->StartsWith(newFolder)) { - warningMessage = wxString(_("Directories are dependent! Be careful when setting up synchronization rules:\n")) + + warningMessage = wxString(_("Directories are dependent! Be careful when setting up synchronization rules:")) + wxT("\n") + wxT("\"") + *i + wxT("\",\n") + wxT("\"") + newFolder + wxT("\""); return true; @@ -278,11 +277,11 @@ bool foldersHaveDependencies(const std::vector& folderPairs, wxStrin CompareProcess::CompareProcess(const bool traverseSymLinks, - const bool handleDstOnFat32Drives, + const unsigned fileTimeTol, bool& warningDependentFolders, StatusHandler* handler) : - traverseSymbolicLinks(traverseSymLinks), - handleDstOnFat32(handleDstOnFat32Drives), + traverseDirectorySymlinks(traverseSymLinks), + fileTimeTolerance(fileTimeTol), m_warningDependentFolders(warningDependentFolders), statusUpdater(handler), txtComparingContentOfFiles(_("Comparing content of files %x")) @@ -312,13 +311,36 @@ struct MemoryAllocator delete [] buffer2; } - static const unsigned int bufferSize = 1024 * 512; //512 kb seems to be the perfect buffer size + static const unsigned int bufferSize = 512 * 1024; //512 kb seems to be the perfect buffer size unsigned char* buffer1; unsigned char* buffer2; }; -bool filesHaveSameContent(const Zstring& filename1, const Zstring& filename2) +//callback functionality +struct CallBackData +{ + StatusHandler* handler; + wxULongLong bytesComparedLast; +}; + +//callback function for status updates whily comparing +typedef void (*CompareCallback)(const wxULongLong&, void*); + + +void compareContentCallback(const wxULongLong& totalBytesTransferred, void* data) +{ //called every 512 kB + CallBackData* sharedData = static_cast(data); + + //inform about the (differential) processed amount of data + sharedData->handler->updateProcessedData(0, (totalBytesTransferred - sharedData->bytesComparedLast).ToDouble()); + sharedData->bytesComparedLast = totalBytesTransferred; + + sharedData->handler->requestUiRefresh(); //exceptions may be thrown here! +} + + +bool filesHaveSameContent(const Zstring& filename1, const Zstring& filename2, CompareCallback callback, void* data) { static MemoryAllocator memory; @@ -330,16 +352,22 @@ bool filesHaveSameContent(const Zstring& filename1, const Zstring& filename2) if (!file2.IsOpened()) //NO cleanup necessary for (wxFFile) file1 throw FileError(Zstring(_("Error reading file:")) + wxT(" \"") + filename2 + wxT("\"")); + wxULongLong bytesCompared; do { - size_t length1 = file1.Read(memory.buffer1, memory.bufferSize); + const size_t length1 = file1.Read(memory.buffer1, memory.bufferSize); if (file1.Error()) throw FileError(Zstring(_("Error reading file:")) + wxT(" \"") + filename1 + wxT("\"")); - size_t length2 = file2.Read(memory.buffer2, memory.bufferSize); + const size_t length2 = file2.Read(memory.buffer2, memory.bufferSize); if (file2.Error()) throw FileError(Zstring(_("Error reading file:")) + wxT(" \"") + filename2 + wxT("\"")); if (length1 != length2 || memcmp(memory.buffer1, memory.buffer2, length1) != 0) return false; + + bytesCompared += length1 * 2; + + //send progress updates + callback(bytesCompared, data); } while (!file1.Eof()); @@ -350,6 +378,34 @@ bool filesHaveSameContent(const Zstring& filename1, const Zstring& filename2) } +bool filesHaveSameContentUpdating(const Zstring& filename1, const Zstring& filename2, const wxULongLong& totalBytesToCmp, StatusHandler* handler) +{ + CallBackData sharedData; + sharedData.handler = handler; +//data.bytesTransferredLast = //amount of bytes that have been compared and communicated to status handler + + bool sameContent = true; + try + { + sameContent = filesHaveSameContent(filename1, filename2, compareContentCallback, &sharedData); + } + catch (...) + { + //error situation: undo communication of processed amount of data + handler->updateProcessedData(0, sharedData.bytesComparedLast.ToDouble() * -1 ); + + throw; + } + + //inform about the (remaining) processed amount of data + handler->updateProcessedData(0, (totalBytesToCmp - sharedData.bytesComparedLast).ToDouble()); + + return sameContent; +} + + +/* OLD IMPLEMENTATION USING A WORKER THREAD + //handle execution of a method while updating the UI class UpdateWhileComparing : public UpdateWhileExecuting { @@ -380,7 +436,7 @@ private: }; -bool filesHaveSameContentMultithreaded(const Zstring& filename1, const Zstring& filename2, StatusHandler* updateClass) +bool filesHaveSameContentUpdating(const Zstring& filename1, const Zstring& filename2, const wxULongLong& totalBytesToCmp, StatusHandler* updateClass) { static UpdateWhileComparing cmpAndUpdate; //single instantiation: thread enters wait phase after each execution @@ -396,11 +452,14 @@ bool filesHaveSameContentMultithreaded(const Zstring& filename1, const Zstring& if (!cmpAndUpdate.success) throw FileError(cmpAndUpdate.errorMessage); + //inform about the processed amount of data + updateClass->updateProcessedData(0, totalBytesToCmp.ToDouble()); + return cmpAndUpdate.sameContent; -} +}*/ -void calcTotalDataForCompare(int& objectsTotal, double& dataTotal, const FileCompareResult& grid, const std::set& rowsToCompare) +void getBytesToCompare(int& objectsTotal, double& dataTotal, const FileCompareResult& grid, const std::set& rowsToCompare) { dataTotal = 0; @@ -417,7 +476,7 @@ void calcTotalDataForCompare(int& objectsTotal, double& dataTotal, const FileCom inline -bool sameFileTime(const time_t a, const time_t b, const time_t tolerance) +bool sameFileTime(const wxLongLong& a, const wxLongLong& b, const unsigned tolerance) { if (a < b) return b - a <= tolerance; @@ -434,6 +493,8 @@ void CompareProcess::startCompareProcess(const std::vector& director wxLogNull noWxLogs; //hide wxWidgets log messages in release build #endif + //PERF_START; + //format directory pairs std::vector directoryPairsFormatted; @@ -505,23 +566,13 @@ void CompareProcess::compareByTimeSize(const std::vector& directoryP //do basis scan: only result lines of type FILE_UNDEFINED (files that exist on both sides) need to be determined after this call this->performBaseComparison(*pair, output); - //add some tolerance if one of the folders is FAT/FAT32 - time_t tolerance = 0; -#ifdef FFS_WIN - if (handleDstOnFat32 && (isFatDrive(pair->leftDirectory) || isFatDrive(pair->rightDirectory))) - tolerance = FILE_TIME_PRECISION + 3600; //tolerate filetime diff <= 1 h to handle daylight saving time issues - else - tolerance = FILE_TIME_PRECISION; -#elif defined FFS_LINUX - tolerance = FILE_TIME_PRECISION; -#endif //categorize files that exist on both sides for (FileCompareResult::iterator i = output.begin() + tableSizeOld; i != output.end(); ++i) { if (i->cmpResult == FILE_UNDEFINED) { //last write time may differ by up to 2 seconds (NTFS vs FAT32) - if (sameFileTime(i->fileDescrLeft.lastWriteTimeRaw, i->fileDescrRight.lastWriteTimeRaw, tolerance)) + if (sameFileTime(i->fileDescrLeft.lastWriteTimeRaw, i->fileDescrRight.lastWriteTimeRaw, fileTimeTolerance)) { if (i->fileDescrLeft.fileSize == i->fileDescrRight.fileSize) i->cmpResult = FILE_EQUAL; @@ -545,6 +596,8 @@ void CompareProcess::compareByTimeSize(const std::vector& directoryP void CompareProcess::compareByContent(const std::vector& directoryPairsFormatted, FileCompareResult& output) { + //PERF_START; + //inform about the total amount of data that will be processed from now on statusUpdater->initNewProcess(-1, 0, StatusHandler::PROCESS_SCANNING); //it's not known how many files will be scanned => -1 objects @@ -573,7 +626,7 @@ void CompareProcess::compareByContent(const std::vector& directoryPa int objectsTotal = 0; double dataTotal = 0; - calcTotalDataForCompare(objectsTotal, dataTotal, output, rowsToCompareBytewise); + getBytesToCompare(objectsTotal, dataTotal, output, rowsToCompareBytewise); statusUpdater->initNewProcess(objectsTotal, dataTotal, StatusHandler::PROCESS_COMPARING_CONTENT); @@ -585,7 +638,7 @@ void CompareProcess::compareByContent(const std::vector& directoryPa FileCompareLine& gridline = output[*i]; Zstring statusText = txtComparingContentOfFiles; - statusText.Replace(wxT("%x"), gridline.fileDescrLeft.relativeName, false); + statusText.Replace(wxT("%x"), gridline.fileDescrLeft.relativeName.c_str(), false); statusUpdater->updateStatusText(statusText); //check files that exist in left and right model but have different content @@ -596,12 +649,12 @@ void CompareProcess::compareByContent(const std::vector& directoryPa try { - if (filesHaveSameContentMultithreaded(gridline.fileDescrLeft.fullName, gridline.fileDescrRight.fullName, statusUpdater)) + if (filesHaveSameContentUpdating(gridline.fileDescrLeft.fullName, gridline.fileDescrRight.fullName, gridline.fileDescrLeft.fileSize * 2, statusUpdater)) gridline.cmpResult = FILE_EQUAL; else gridline.cmpResult = FILE_DIFFERENT; - statusUpdater->updateProcessedData(2, (gridline.fileDescrLeft.fileSize * 2).ToDouble()); + statusUpdater->updateProcessedData(2, 0); //processed data is communicated in subfunctions! break; } catch (FileError& error) @@ -694,7 +747,7 @@ void CompareProcess::performBaseComparison(const FolderPair& pair, FileCompareRe } //PERF_STOP; - //reserve some space to avoid too many vector reallocations + //reserve some space to avoid too many vector reallocations: doesn't make much sense for multiple folder pairs, but doesn't hurt either output.reserve(output.size() + unsigned(std::max(directoryLeft->size(), directoryRight->size()) * 1.2)); //begin base comparison -- cgit