// ************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) Zenju (zenju AT gmx DOT de) - All Rights Reserved * // ************************************************************************** #include "comparison.h" #include #include #include #include #include #include #include #include "algorithm.h" #include "lib/parallel_scan.h" #include "lib/resolve_path.h" #include "lib/dir_exist_async.h" #include "lib/binary.h" #include "lib/cmp_filetime.h" #include "lib/status_handler_impl.h" #include "lib/parallel_scan.h" using namespace zen; std::vector zen::extractCompareCfg(const MainConfiguration& mainCfg) { //merge first and additional pairs std::vector allPairs; allPairs.push_back(mainCfg.firstPair); allPairs.insert(allPairs.end(), mainCfg.additionalPairs.begin(), //add additional pairs mainCfg.additionalPairs.end()); std::vector output; std::transform(allPairs.begin(), allPairs.end(), std::back_inserter(output), [&](const FolderPairEnh& enhPair) -> FolderPairCfg { const Zstring leftDirFmt = getFormattedDirectoryName(enhPair.leftDirectory); //ensure they end with FILE_NAME_SEPARATOR and replace macros const Zstring rightDirFmt = getFormattedDirectoryName(enhPair.rightDirectory); // return FolderPairCfg(leftDirFmt, rightDirFmt, enhPair.altCmpConfig.get() ? enhPair.altCmpConfig->compareVar : mainCfg.cmpConfig.compareVar, enhPair.altCmpConfig.get() ? enhPair.altCmpConfig->handleSymlinks : mainCfg.cmpConfig.handleSymlinks, normalizeFilters(mainCfg.globalFilter, enhPair.localFilter), enhPair.altSyncConfig.get() ? enhPair.altSyncConfig->directionCfg : mainCfg.syncCfg.directionCfg); }); return output; } //------------------------------------------------------------------------------------------ namespace { void checkForIncompleteInput(const std::vector& folderPairsForm, bool& warningInputFieldEmpty, ProcessCallback& callback) { bool havePartialPair = false; bool haveFullPair = false; std::for_each(folderPairsForm.begin(), folderPairsForm.end(), [&](const FolderPairCfg& fpCfg) { if (fpCfg.leftDirectoryFmt.empty() != fpCfg.rightDirectoryFmt.empty()) havePartialPair = true; else if (!fpCfg.leftDirectoryFmt.empty()) haveFullPair = true; }); if (havePartialPair == haveFullPair) //error if: all empty or exist both full and partial pairs -> support single-dir scenario callback.reportWarning(_("A folder input field is empty.") + L" \n\n" + _("The corresponding folder will be considered as empty."), warningInputFieldEmpty); } std::set determineExistentDirs(const std::set& dirnames, bool allowUserInteraction, ProcessCallback& callback) { std::set dirsEx; tryReportingError2([&] { dirsEx = getExistingDirsUpdating(dirnames, allowUserInteraction, callback); //check *all* directories on each try! //get list of not existing directories std::set dirsMissing = dirnames; set_remove_if(dirsMissing, [&](const Zstring& dirname) { return dirname.empty() || dirsEx.find(dirname) != dirsEx.end(); }); if (!dirsMissing.empty()) { std::wstring msg = _("Cannot find the following folders:") + L"\n"; std::for_each(dirsMissing.begin(), dirsMissing.end(), [&](const Zstring& dirname) { msg += std::wstring(L"\n") + dirname; }); msg += L"\n\n" + _("You can ignore this error to consider each folder as empty. The folders then will be created automatically during synchronization."); throw FileError(msg); } }, callback); return dirsEx; } //check whether one side is subdirectory of other side (folder pair wise!) //similar check if one directory is read/written by multiple pairs not before beginning of synchronization void checkFolderDependency(const std::vector& folderPairsForm, bool& warningDependentFolders, ProcessCallback& callback) //returns warning message, empty if all ok { std::vector> dependentDirs; auto dependentDir = [](const Zstring& lhs, const Zstring& rhs) { return EqualFilename()(Zstring(lhs.c_str(), std::min(lhs.length(), rhs.length())), //note: this is NOT an equivalence relation! Zstring(rhs.c_str(), std::min(lhs.length(), rhs.length()))); }; for (std::vector::const_iterator i = folderPairsForm.begin(); i != folderPairsForm.end(); ++i) if (!i->leftDirectoryFmt.empty() && !i->rightDirectoryFmt.empty()) //empty folders names may be accepted by user { if (dependentDir(i->leftDirectoryFmt, i->rightDirectoryFmt)) //test wheter leftDirectory begins with rightDirectory or the other way round dependentDirs.push_back(std::make_pair(i->leftDirectoryFmt, i->rightDirectoryFmt)); } if (!dependentDirs.empty()) { std::wstring warningMsg = _("Directories are dependent! Be careful when setting up synchronization rules:"); for (auto it = dependentDirs.begin(); it != dependentDirs.end(); ++it) warningMsg += std::wstring(L"\n\n") + it->first + L"\n" + it->second; callback.reportWarning(warningMsg, warningDependentFolders); } } //callback implementation class CmpCallbackImpl : public CompareCallback { public: CmpCallbackImpl(ProcessCallback& pc, Int64& bytesReported) : pc_(pc), bytesReported_(bytesReported) {} virtual void updateCompareStatus(Int64 bytesDelta) { //inform about the (differential) processed amount of data pc_.updateProcessedData(0, bytesDelta); //throw()! -> ensure client and service provider are in sync! bytesReported_ += bytesDelta; // pc_.requestUiRefresh(); //may throw } private: ProcessCallback& pc_; Int64& bytesReported_; }; bool filesHaveSameContentUpdating(const Zstring& filename1, const Zstring& filename2, Int64 expectedBytesToCmp, ProcessCallback& pc) //throw FileError { Int64 bytesReported; //amount of bytes that have been compared and communicated to status handler //error = unexpected increase of total workload zen::ScopeGuard guardStatistics = zen::makeGuard([&] { pc.updateTotalData(0, bytesReported); }); CmpCallbackImpl callback(pc, bytesReported); bool sameContent = filesHaveSameContent(filename1, filename2, callback); //throw FileError guardStatistics.dismiss(); //update statistics to consider the real amount of data processed: consider short-cut behavior if first bytes differ! if (bytesReported != expectedBytesToCmp) pc.updateTotalData(0, bytesReported - expectedBytesToCmp); return sameContent; } //############################################################################################################################# class ComparisonBuffer { public: ComparisonBuffer(const std::set& keysToRead, size_t fileTimeTol, ProcessCallback& callback); //create comparison result table and fill category except for files existing on both sides: undefinedFiles and undefinedLinks are appended! void compareByTimeSize(const FolderPairCfg& fpConfig, BaseDirMapping& output); void compareByContent(std::vector>& workLoad); private: ComparisonBuffer(const ComparisonBuffer&); ComparisonBuffer& operator=(const ComparisonBuffer&); void performComparison(const FolderPairCfg& fpCfg, BaseDirMapping& output, std::vector& undefinedFiles, std::vector& undefinedLinks); std::map directoryBuffer; //contains only *existing* directories const size_t fileTimeTolerance; ProcessCallback& callback_; }; ComparisonBuffer::ComparisonBuffer(const std::set& keysToRead, size_t fileTimeTol, ProcessCallback& callback) : fileTimeTolerance(fileTimeTol), callback_(callback) { class CbImpl : public FillBufferCallback { public: CbImpl(ProcessCallback& pcb) : callback_(pcb), itemsReported(0) {} virtual void reportStatus(const std::wstring& statusMsg, int itemsTotal) { callback_.updateProcessedData(itemsTotal - itemsReported, 0); //processed bytes are reported in subfunctions! itemsReported = itemsTotal; callback_.reportStatus(statusMsg); //may throw //callback_.requestUiRefresh(); //already called by reportStatus() } virtual HandleError reportError(const std::wstring& msg) { switch (callback_.reportError(msg)) { case ProcessCallback::IGNORE_ERROR: return ON_ERROR_IGNORE; case ProcessCallback::RETRY: return ON_ERROR_RETRY; } assert(false); return ON_ERROR_IGNORE; } private: ProcessCallback& callback_; int itemsReported; } cb(callback); fillBuffer(keysToRead, //in directoryBuffer, //out cb, UI_UPDATE_INTERVAL / 2); //every ~50 ms } //--------------------assemble conflict descriptions--------------------------- //const wchar_t arrowLeft [] = L"\u2190"; //const wchar_t arrowRight[] = L"\u2192"; unicode arrows -> too small const wchar_t arrowLeft [] = L"<--"; const wchar_t arrowRight[] = L"-->"; //check for very old dates or date2s in the future std::wstring getConflictInvalidDate(const Zstring& fileNameFull, Int64 utcTime) { return _("Conflict detected:") + L" " + replaceCpy(_("File %x has an invalid date!"), L"%x", fmtFileName(fileNameFull)) + L"\n" + _("Date:") + L" " + utcToLocalTimeString(utcTime); } //check for changed files with same modification date std::wstring getConflictSameDateDiffSize(const FileMapping& fileObj) { return _("Conflict detected:") + L" " + replaceCpy(_("Files %x have the same date but a different size!"), L"%x", fmtFileName(fileObj.getObjRelativeName())) + L"\n" + arrowLeft + L" " + _("Date:") + L" " + utcToLocalTimeString(fileObj.getLastWriteTime()) + L" " + _("Size:") + L" " + toGuiString(fileObj.getFileSize()) + L"\n" + arrowRight + L" " + _("Date:") + L" " + utcToLocalTimeString(fileObj.getLastWriteTime()) + L" " + _("Size:") + L" " + toGuiString(fileObj.getFileSize()); } inline std::wstring getDescrDiffMetaShortnameCase(const FileSystemObject& fsObj) { return _("Items differ in attributes only") + L"\n" + arrowLeft + L" " + fmtFileName(fsObj.getShortName()) + L"\n" + arrowRight + L" " + fmtFileName(fsObj.getShortName()); } template inline std::wstring getDescrDiffMetaDate(const FileOrLinkMapping& fileObj) { return _("Items differ in attributes only") + L"\n" + arrowLeft + L" " + _("Date:") + L" " + utcToLocalTimeString(fileObj.template getLastWriteTime()) + L"\n" + arrowRight + L" " + _("Date:") + L" " + utcToLocalTimeString(fileObj.template getLastWriteTime()); } //----------------------------------------------------------------------------- void categorizeSymlinkByTime(SymLinkMapping& linkObj, size_t fileTimeTolerance) { //categorize symlinks that exist on both sides switch (CmpFileTime::getResult(linkObj.getLastWriteTime(), linkObj.getLastWriteTime(), fileTimeTolerance)) { case CmpFileTime::TIME_EQUAL: //Caveat: //1. SYMLINK_EQUAL may only be set if short names match in case: InSyncDir's mapping tables use short name as a key! see db_file.cpp //2. harmonize with "bool stillInSync()" in algorithm.cpp if (linkObj.getShortName() == linkObj.getShortName()) linkObj.setCategory(); else linkObj.setCategoryDiffMetadata(getDescrDiffMetaShortnameCase(linkObj)); break; case CmpFileTime::TIME_LEFT_NEWER: linkObj.setCategory(); break; case CmpFileTime::TIME_RIGHT_NEWER: linkObj.setCategory(); break; case CmpFileTime::TIME_LEFT_INVALID: linkObj.setCategoryConflict(getConflictInvalidDate(linkObj.getFullName(), linkObj.getLastWriteTime())); break; case CmpFileTime::TIME_RIGHT_INVALID: linkObj.setCategoryConflict(getConflictInvalidDate(linkObj.getFullName(), linkObj.getLastWriteTime())); break; } } void ComparisonBuffer::compareByTimeSize(const FolderPairCfg& fpConfig, BaseDirMapping& output) { //do basis scan and retrieve files existing on both sides as "compareCandidates" std::vector uncategorizedFiles; std::vector uncategorizedLinks; performComparison(fpConfig, output, uncategorizedFiles, uncategorizedLinks); //finish symlink categorization std::for_each(uncategorizedLinks.begin(), uncategorizedLinks.end(), [&](SymLinkMapping* linkMap) { categorizeSymlinkByTime(*linkMap, fileTimeTolerance); }); //categorize files that exist on both sides std::for_each(uncategorizedFiles.begin(), uncategorizedFiles.end(), [&](FileMapping* fileObj) { switch (CmpFileTime::getResult(fileObj->getLastWriteTime(), fileObj->getLastWriteTime(), fileTimeTolerance)) { case CmpFileTime::TIME_EQUAL: if (fileObj->getFileSize() == fileObj->getFileSize()) { //Caveat: //1. FILE_EQUAL may only be set if short names match in case: InSyncDir's mapping tables use short name as a key! see db_file.cpp //2. harmonize with "bool stillInSync()" in algorithm.cpp if (fileObj->getShortName() == fileObj->getShortName()) fileObj->setCategory(); else fileObj->setCategoryDiffMetadata(getDescrDiffMetaShortnameCase(*fileObj)); } else fileObj->setCategoryConflict(getConflictSameDateDiffSize(*fileObj)); //same date, different filesize break; case CmpFileTime::TIME_LEFT_NEWER: fileObj->setCategory(); break; case CmpFileTime::TIME_RIGHT_NEWER: fileObj->setCategory(); break; case CmpFileTime::TIME_LEFT_INVALID: fileObj->setCategoryConflict(getConflictInvalidDate(fileObj->getFullName(), fileObj->getLastWriteTime())); break; case CmpFileTime::TIME_RIGHT_INVALID: fileObj->setCategoryConflict(getConflictInvalidDate(fileObj->getFullName(), fileObj->getLastWriteTime())); break; } }); } void categorizeSymlinkByContent(SymLinkMapping& linkObj, size_t fileTimeTolerance, ProcessCallback& callback) { //categorize symlinks that exist on both sides Zstring targetPathRawL; Zstring targetPathRawR; Opt errMsg = tryReportingError2([&] { callback.reportStatus(replaceCpy(_("Resolving symbolic link %x"), L"%x", fmtFileName(linkObj.getFullName()))); targetPathRawL = getSymlinkTargetRaw(linkObj.getFullName()); //throw FileError callback.reportStatus(replaceCpy(_("Resolving symbolic link %x"), L"%x", fmtFileName(linkObj.getFullName()))); targetPathRawR = getSymlinkTargetRaw(linkObj.getFullName()); //throw FileError }, callback); if (errMsg) linkObj.setCategoryConflict(_("Conflict detected:") + L" " + *errMsg); else { if (targetPathRawL == targetPathRawR #ifdef FFS_WIN //type of symbolic link is relevant for Windows only && getSymlinkType(linkObj.getFullName()) == getSymlinkType(linkObj.getFullName()) #endif ) { //Caveat: //1. SYMLINK_EQUAL may only be set if short names match in case: InSyncDir's mapping tables use short name as a key! see db_file.cpp //2. harmonize with "bool stillInSync()" in algorithm.cpp //symlinks have same "content" if (linkObj.getShortName() != linkObj.getShortName()) linkObj.setCategoryDiffMetadata(getDescrDiffMetaShortnameCase(linkObj)); else if (CmpFileTime::getResult(linkObj.getLastWriteTime(), linkObj.getLastWriteTime(), fileTimeTolerance) != CmpFileTime::TIME_EQUAL) linkObj.setCategoryDiffMetadata(getDescrDiffMetaDate(linkObj)); else linkObj.setCategory(); } else linkObj.setCategory(); } } void ComparisonBuffer::compareByContent(std::vector>& workLoad) { if (workLoad.empty()) return; //PERF_START; std::vector undefinedFiles; //process one folder pair after each other for (auto workItem = workLoad.begin(); workItem != workLoad.end(); ++workItem) { std::vector uncategorizedLinks; //do basis scan and retrieve candidates for binary comparison (files existing on both sides) performComparison(workItem->first, *workItem->second, undefinedFiles, uncategorizedLinks); //finish symlink categorization std::for_each(uncategorizedLinks.begin(), uncategorizedLinks.end(), [&](SymLinkMapping* linkMap) { categorizeSymlinkByContent(*linkMap, fileTimeTolerance, callback_); }); } //finish categorization... std::vector filesToCompareBytewise; //content comparison of file content happens AFTER finding corresponding files //in order to separate into two processes (scanning and comparing) std::for_each(undefinedFiles.begin(), undefinedFiles.end(), [&](FileMapping* fileObj) { //pre-check: files have different content if they have a different filesize if (fileObj->getFileSize() != fileObj->getFileSize()) fileObj->setCategory(); else filesToCompareBytewise.push_back(fileObj); }); const size_t objectsTotal = filesToCompareBytewise.size(); const UInt64 bytesTotal = //left and right filesizes are equal std::accumulate(filesToCompareBytewise.begin(), filesToCompareBytewise.end(), static_cast(0), [](UInt64 sum, FileMapping* fsObj) { return sum + fsObj->getFileSize(); }); callback_.initNewPhase(static_cast(objectsTotal), to(bytesTotal), ProcessCallback::PHASE_COMPARING_CONTENT); const std::wstring txtComparingContentOfFiles = _("Comparing content of files %x"); //compare files (that have same size) bytewise... std::for_each(filesToCompareBytewise.begin(), filesToCompareBytewise.end(), [&](FileMapping* fileObj) { callback_.reportStatus(replaceCpy(txtComparingContentOfFiles, L"%x", fmtFileName(fileObj->getObjRelativeName()), false)); //check files that exist in left and right model but have different content bool haveSameContent = false; Opt errMsg = tryReportingError2([&] { haveSameContent = filesHaveSameContentUpdating(fileObj->getFullName(), //throw FileError fileObj->getFullName(), to(fileObj->getFileSize()), callback_); callback_.updateProcessedData(1, 0); //processed bytes are reported in subfunctions! callback_.requestUiRefresh(); //may throw }, callback_); if (errMsg) fileObj->setCategoryConflict(_("Conflict detected:") + L" " + *errMsg); else { if (haveSameContent) { //Caveat: //1. FILE_EQUAL may only be set if short names match in case: InSyncDir's mapping tables use short name as a key! see db_file.cpp //2. harmonize with "bool stillInSync()" in algorithm.cpp if (fileObj->getShortName() != fileObj->getShortName()) fileObj->setCategoryDiffMetadata(getDescrDiffMetaShortnameCase(*fileObj)); else if (CmpFileTime::getResult(fileObj->getLastWriteTime(), fileObj->getLastWriteTime(), fileTimeTolerance) != CmpFileTime::TIME_EQUAL) fileObj->setCategoryDiffMetadata(getDescrDiffMetaDate(*fileObj)); else fileObj->setCategory(); } else fileObj->setCategory(); } }); } class MergeSides { public: MergeSides(std::vector& appendUndefinedFileOut, std::vector& appendUndefinedLinkOut) : appendUndefinedFile(appendUndefinedFileOut), appendUndefinedLink(appendUndefinedLinkOut) {} void execute(const DirContainer& leftSide, const DirContainer& rightSide, HierarchyObject& output); private: template void fillOneSide(const DirContainer& dirCont, HierarchyObject& output); std::vector& appendUndefinedFile; std::vector& appendUndefinedLink; }; template void MergeSides::fillOneSide(const DirContainer& dirCont, HierarchyObject& output) { for (auto it = dirCont.files.cbegin(); it != dirCont.files.cend(); ++it) output.addSubFile(it->first, it->second); for (auto it = dirCont.links.cbegin(); it != dirCont.links.cend(); ++it) output.addSubLink(it->first, it->second); for (auto it = dirCont.dirs.cbegin(); it != dirCont.dirs.cend(); ++it) { DirMapping& newDirMap = output.addSubDir(it->first); fillOneSide(it->second, newDirMap); //recurse } } //improve merge-perf by over 70% + more natural default sequence template inline void linearMerge(const MapType& mapLeft, const MapType& mapRight, ProcessLeftOnly lo, ProcessRightOnly ro, ProcessBoth bo) { const auto lessVal = mapLeft.value_comp(); auto iterLeft = mapLeft .begin(); auto iterRight = mapRight.begin(); auto finishLeft = [&] { std::for_each(iterLeft, mapLeft .end(), lo); }; auto finishRight = [&] { std::for_each(iterRight, mapRight.end(), ro); }; if (iterLeft == mapLeft .end()) return finishRight(); if (iterRight == mapRight.end()) return finishLeft(); for (;;) if (lessVal(*iterLeft, *iterRight)) { lo(*iterLeft); if (++iterLeft == mapLeft.end()) return finishRight(); } else if (lessVal(*iterRight, *iterLeft)) { ro(*iterRight); if (++iterRight == mapRight.end()) return finishLeft(); } else { bo(*iterLeft, *iterRight); ++iterLeft; // ++iterRight; //increment BOTH before checking for end of range! if (iterLeft == mapLeft .end()) return finishRight(); if (iterRight == mapRight.end()) return finishLeft(); } } void MergeSides::execute(const DirContainer& leftSide, const DirContainer& rightSide, HierarchyObject& output) { //HierarchyObject::addSubFile() must NOT invalidate references used in "appendUndefined"! typedef const DirContainer::FileList::value_type FileData; linearMerge(leftSide.files, rightSide.files, [&](const FileData& fileLeft ) { output.addSubFile(fileLeft .first, fileLeft .second); }, //left only [&](const FileData& fileRight) { output.addSubFile(fileRight.first, fileRight.second); }, //right only [&](const FileData& fileLeft, const FileData& fileRight) //both sides { FileMapping& newEntry = output.addSubFile(fileLeft.first, fileLeft.second, FILE_EQUAL, //FILE_EQUAL is just a dummy-value here fileRight.first, fileRight.second); appendUndefinedFile.push_back(&newEntry); }); //----------------------------------------------------------------------------------------------- typedef const DirContainer::LinkList::value_type LinkData; linearMerge(leftSide.links, rightSide.links, [&](const LinkData& linkLeft) { output.addSubLink(linkLeft.first, linkLeft.second); }, //left only [&](const LinkData& linkRight) { output.addSubLink(linkRight.first, linkRight.second); }, //right only [&](const LinkData& linkLeft, const LinkData& linkRight) //both sides { SymLinkMapping& newEntry = output.addSubLink(linkLeft.first, linkLeft.second, SYMLINK_EQUAL, //SYMLINK_EQUAL is just a dummy-value here linkRight.first, linkRight.second); appendUndefinedLink.push_back(&newEntry); }); //----------------------------------------------------------------------------------------------- typedef const DirContainer::DirList::value_type DirData; linearMerge(leftSide.dirs, rightSide.dirs, [&](const DirData& dirLeft) //left only { DirMapping& newDirMap = output.addSubDir(dirLeft.first); this->fillOneSide(dirLeft.second, newDirMap); //recurse into subdirectories }, [&](const DirData& dirRight) //right only { DirMapping& newDirMap = output.addSubDir(dirRight.first); this->fillOneSide(dirRight.second, newDirMap); //recurse into subdirectories }, [&](const DirData& dirLeft, const DirData& dirRight) //both sides { DirMapping& newDirMap = output.addSubDir(dirLeft.first, dirRight.first, DIR_EQUAL); if (dirLeft.first != dirRight.first) newDirMap.setCategoryDiffMetadata(getDescrDiffMetaShortnameCase(newDirMap)); execute(dirLeft.second, dirRight.second, newDirMap); //recurse into subdirectories }); } //mark excluded directories (see fillBuffer()) + remove superfluous excluded subdirectories //note: this cannot be done while traversing directory, since both sides need to be taken into account, both for filtering AND removing subdirs! void processFilteredDirs(HierarchyObject& hierObj, const HardFilter& filterProc) { auto& subDirs = hierObj.refSubDirs(); //process subdirs recursively std::for_each(subDirs.begin(), subDirs.end(), [&](DirMapping& dirObj) { dirObj.setActive(filterProc.passDirFilter(dirObj.getObjRelativeName(), nullptr)); //subObjMightMatch is always true in this context! processFilteredDirs(dirObj, filterProc); }); //remove superfluous directories -> note: this does not invalidate "std::vector& undefinedFiles", since we delete folders only //and there is no side-effect for memory positions of FileMapping and SymlinkMapping thanks to zen::FixedList! subDirs.remove_if([](DirMapping& dirObj) { return !dirObj.isActive() && dirObj.refSubDirs ().empty() && dirObj.refSubLinks().empty() && dirObj.refSubFiles().empty(); }); } //create comparison result table and fill category except for files existing on both sides: undefinedFiles and undefinedLinks are appended! void ComparisonBuffer::performComparison(const FolderPairCfg& fpCfg, BaseDirMapping& output, std::vector& undefinedFiles, std::vector& undefinedLinks) { assert(output.refSubDirs(). empty()); assert(output.refSubLinks().empty()); assert(output.refSubFiles().empty()); //PERF_START; auto getDirValue = [&](const Zstring& dirnameFmt) -> const DirectoryValue* { auto it = directoryBuffer.find(DirectoryKey(dirnameFmt, fpCfg.filter.nameFilter, fpCfg.handleSymlinks)); return it != directoryBuffer.end() ? &it->second : nullptr; }; const DirectoryValue* bufValueLeft = getDirValue(fpCfg.leftDirectoryFmt); const DirectoryValue* bufValueRight = getDirValue(fpCfg.rightDirectoryFmt); callback_.reportStatus(_("Generating file list...")); callback_.forceUiRefresh(); //PERF_START; MergeSides(undefinedFiles, undefinedLinks).execute(bufValueLeft ? bufValueLeft ->dirCont : DirContainer(), bufValueRight ? bufValueRight->dirCont : DirContainer(), output); //PERF_STOP; //##################### in/exclude rows according to filtering ##################### //attention: some excluded directories are still in the comparison result! (see include filter handling!) if (!fpCfg.filter.nameFilter->isNull()) processFilteredDirs(output, *fpCfg.filter.nameFilter); //mark excluded directories (see fillBuffer()) + remove superfluous excluded subdirectories //apply soft filtering / hard filter already applied addSoftFiltering(output, fpCfg.filter.timeSizeFilter); //properly handle (user-ignored) traversing errors: just uncheck them, no need to physically delete them ( mode will be grateful) auto addToSet = [](std::set& s, const std::set& v) { s.insert(v.begin(), v.end()); }; std::set failedDirReads; if (bufValueLeft ) addToSet(failedDirReads, bufValueLeft ->failedDirReads); if (bufValueRight) addToSet(failedDirReads, bufValueRight->failedDirReads); std::set failedItemReads; if (bufValueLeft ) addToSet(failedItemReads, bufValueLeft ->failedItemReads); if (bufValueRight) addToSet(failedItemReads, bufValueRight->failedItemReads); if (!failedDirReads.empty() || !failedItemReads.empty()) { Zstring filterFailedRead; //note: relDirPf is empty for base dir, otherwise postfixed! e.g. "subdir\" std::for_each(failedDirReads .begin(), failedDirReads .end(), [&](const Zstring& relDirPf) { filterFailedRead += relDirPf + Zstr("?*\n"); }); //exclude child items only! std::for_each(failedItemReads.begin(), failedItemReads.end(), [&](const Zstring& relItem ) { filterFailedRead += relItem + Zstr("\n"); }); //exclude item AND child items! addHardFiltering(output, filterFailedRead); } //################################################################################## } } void zen::compare(size_t fileTimeTolerance, xmlAccess::OptionalDialogs& warnings, bool allowUserInteraction, bool runWithBackgroundPriority, bool createDirLocks, std::unique_ptr& dirLocks, const std::vector& cfgList, FolderComparison& output, ProcessCallback& callback) { //specify process and resource handling priorities std::unique_ptr backgroundPrio; if (runWithBackgroundPriority) try { backgroundPrio = make_unique(); //throw FileError } catch (const FileError& e) { //not an error in this context callback.reportInfo(e.toString()); //may throw! } //prevent operating system going into sleep state std::unique_ptr noStandby; try { noStandby = make_unique(); //throw FileError } catch (const FileError& e) { //not an error in this context callback.reportInfo(e.toString()); //may throw! } //PERF_START; callback.reportInfo(_("Start comparison")); //we want some indicator at the very beginning to make sense of "total time" //init process: keep at beginning so that all gui elements are initialized properly callback.initNewPhase(-1, 0, ProcessCallback::PHASE_SCANNING); //it's not known how many files will be scanned => -1 objects //-------------------some basic checks:------------------------------------------ checkForIncompleteInput(cfgList, warnings.warningInputFieldEmpty, callback); checkFolderDependency (cfgList, warnings.warningDependentFolders, callback); std::set dirnamesExisting; //list of directories that are *expected* to be existent (and need to be scanned)! //directory existence only checked *once* to avoid race conditions! { std::set dirnames; std::for_each(cfgList.begin(), cfgList.end(), [&](const FolderPairCfg& fpCfg) { dirnames.insert(fpCfg.leftDirectoryFmt); dirnames.insert(fpCfg.rightDirectoryFmt); }); dirnamesExisting = determineExistentDirs(dirnames, allowUserInteraction, callback); } auto dirAvailable = [&](const Zstring& dirnameFmt) { return dirnamesExisting.find(dirnameFmt) != dirnamesExisting.end(); }; //-------------------end of basic checks------------------------------------------ try { //lock (existing) directories before comparison if (createDirLocks) dirLocks = make_unique(dirnamesExisting, warnings.warningDirectoryLockFailed, callback); //------------------- fill directory buffer --------------------------------------------------- std::set keysToRead; std::for_each(cfgList.begin(), cfgList.end(), [&](const FolderPairCfg& fpCfg) { if (dirAvailable(fpCfg.leftDirectoryFmt)) //only request *currently existing* directories: at this point user is aware that non-ex + empty string are seen as empty folder! keysToRead.insert(DirectoryKey(fpCfg.leftDirectoryFmt, fpCfg.filter.nameFilter, fpCfg.handleSymlinks)); if (dirAvailable(fpCfg.rightDirectoryFmt)) keysToRead.insert(DirectoryKey(fpCfg.rightDirectoryFmt, fpCfg.filter.nameFilter, fpCfg.handleSymlinks)); }); ComparisonBuffer cmpBuff(keysToRead, fileTimeTolerance, callback); //------------ traverse/read folders ----------------------------------------------------- FolderComparison output_tmp; //write to output not before END of process! //buffer "config"/"result of binary comparison" for latter processing as a single block std::vector> workLoadBinary; std::for_each(cfgList.begin(), cfgList.end(), [&](const FolderPairCfg& fpCfg) { //a pity VC11 screws up on std::make_shared with 7 arguments... output_tmp.push_back(std::shared_ptr(new BaseDirMapping(fpCfg.leftDirectoryFmt, dirAvailable(fpCfg.leftDirectoryFmt), fpCfg.rightDirectoryFmt, dirAvailable(fpCfg.rightDirectoryFmt), fpCfg.filter.nameFilter, fpCfg.compareVar, fileTimeTolerance))); switch (fpCfg.compareVar) { case CMP_BY_TIME_SIZE: cmpBuff.compareByTimeSize(fpCfg, *output_tmp.back()); break; case CMP_BY_CONTENT: workLoadBinary.push_back(std::make_pair(fpCfg, &*output_tmp.back())); break; } }); //process binary comparison in one block cmpBuff.compareByContent(workLoadBinary); assert(output_tmp.size() == cfgList.size()); //--------- set initial sync-direction -------------------------------------------------- for (auto j = begin(output_tmp); j != end(output_tmp); ++j) { const FolderPairCfg& fpCfg = cfgList[j - output_tmp.begin()]; callback.reportStatus(_("Calculating sync directions...")); callback.forceUiRefresh(); zen::redetermineSyncDirection(fpCfg.directionCfg, *j, [&](const std::wstring& warning) { callback.reportWarning(warning, warnings.warningDatabaseError); }); } //only if everything was processed correctly output is written to! //note: output mustn't change during this process to be in sync with GUI grid view!!! output_tmp.swap(output); } catch (const std::bad_alloc& e) { callback.reportFatalError(_("Out of memory!") + L" " + utfCvrtTo(e.what())); } catch (const std::exception& e) { callback.reportFatalError(utfCvrtTo(e.what())); } }