// ************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) Zenju (zenju AT gmx DOT de) - All Rights Reserved * // ************************************************************************** #include "synchronization.h" #include #include #include #include //get rid!? #include #include #include #include #include #include "lib/resolve_path.h" #include "lib/db_file.h" #include "lib/dir_exist_async.h" #include "lib/cmp_filetime.h" #include #include #include "lib/status_handler_impl.h" #include "lib/versioning.h" #ifdef FFS_WIN #include #include #include "lib/shadow.h" #endif using namespace zen; namespace { inline int getCUD(const SyncStatistics& stat) { return stat.getCreate() + stat.getUpdate() + stat.getDelete(); } } void SyncStatistics::init() { createLeft = 0; createRight = 0; updateLeft = 0; updateRight = 0; deleteLeft = 0; deleteRight = 0; //conflict = 0; rowsTotal = 0; } SyncStatistics::SyncStatistics(const FolderComparison& folderCmp) { init(); std::for_each(begin(folderCmp), end(folderCmp), [&](const BaseDirMapping& baseMap) { recurse(baseMap); }); } SyncStatistics::SyncStatistics(const HierarchyObject& hierObj) { init(); recurse(hierObj); } SyncStatistics::SyncStatistics(const FileMapping& fileObj) { init(); calcStats(fileObj); rowsTotal += 1; } inline void SyncStatistics::recurse(const HierarchyObject& hierObj) { std::for_each(hierObj.refSubDirs ().begin(), hierObj.refSubDirs ().end(), [&](const DirMapping& dirObj ) { calcStats(dirObj ); }); std::for_each(hierObj.refSubFiles().begin(), hierObj.refSubFiles().end(), [&](const FileMapping& fileObj) { calcStats(fileObj); }); std::for_each(hierObj.refSubLinks().begin(), hierObj.refSubLinks().end(), [&](const SymLinkMapping& linkObj) { calcStats(linkObj); }); rowsTotal += hierObj.refSubDirs(). size(); rowsTotal += hierObj.refSubFiles().size(); rowsTotal += hierObj.refSubLinks().size(); } inline void SyncStatistics::calcStats(const FileMapping& fileObj) { switch (fileObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_CREATE_NEW_LEFT: ++createLeft; dataToProcess += to(fileObj.getFileSize()); break; case SO_CREATE_NEW_RIGHT: ++createRight; dataToProcess += to(fileObj.getFileSize()); break; case SO_DELETE_LEFT: ++deleteLeft; break; case SO_DELETE_RIGHT: ++deleteRight; break; case SO_MOVE_LEFT_TARGET: ++updateLeft; break; case SO_MOVE_RIGHT_TARGET: ++updateRight; break; case SO_MOVE_LEFT_SOURCE: //ignore; already counted case SO_MOVE_RIGHT_SOURCE: // break; case SO_OVERWRITE_LEFT: ++updateLeft; dataToProcess += to(fileObj.getFileSize()); break; case SO_OVERWRITE_RIGHT: ++updateRight; dataToProcess += to(fileObj.getFileSize()); break; case SO_UNRESOLVED_CONFLICT: //++conflict; //if (conflictMsgs.size() < MAX_CONFLICTS) //save the first conflict texts conflictMsgs.push_back(std::make_pair(fileObj.getObjRelativeName(), fileObj.getSyncOpConflict())); break; case SO_COPY_METADATA_TO_LEFT: ++updateLeft; break; case SO_COPY_METADATA_TO_RIGHT: ++updateRight; break; case SO_DO_NOTHING: case SO_EQUAL: break; } } inline void SyncStatistics::calcStats(const SymLinkMapping& linkObj) { switch (linkObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_CREATE_NEW_LEFT: ++createLeft; break; case SO_CREATE_NEW_RIGHT: ++createRight; break; case SO_DELETE_LEFT: ++deleteLeft; break; case SO_DELETE_RIGHT: ++deleteRight; break; case SO_OVERWRITE_LEFT: case SO_COPY_METADATA_TO_LEFT: ++updateLeft; break; case SO_OVERWRITE_RIGHT: case SO_COPY_METADATA_TO_RIGHT: ++updateRight; break; case SO_UNRESOLVED_CONFLICT: //++conflict; //if (conflictMsgs.size() < MAX_CONFLICTS) //save the first conflict texts conflictMsgs.push_back(std::make_pair(linkObj.getObjRelativeName(), linkObj.getSyncOpConflict())); break; case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_DO_NOTHING: case SO_EQUAL: break; } } inline void SyncStatistics::calcStats(const DirMapping& dirObj) { switch (dirObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_CREATE_NEW_LEFT: ++createLeft; break; case SO_CREATE_NEW_RIGHT: ++createRight; break; case SO_DELETE_LEFT: //if deletion variant == user-defined directory existing on other volume, this results in a full copy + delete operation! ++deleteLeft; //however we cannot (reliably) anticipate this situation, fortunately statistics can be adapted during sync! break; case SO_DELETE_RIGHT: ++deleteRight; break; case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: assert(false); break; case SO_UNRESOLVED_CONFLICT: //++conflict; //if (conflictMsgs.size() < MAX_CONFLICTS) //save the first conflict texts conflictMsgs.push_back(std::make_pair(dirObj.getObjRelativeName(), dirObj.getSyncOpConflict())); break; case SO_COPY_METADATA_TO_LEFT: ++updateLeft; break; case SO_COPY_METADATA_TO_RIGHT: ++updateRight; break; case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_DO_NOTHING: case SO_EQUAL: break; } recurse(dirObj); //since we model logical stats, we recurse, even if deletion variant is "recycler" or "versioning + same volume", which is a single physical operation! } //----------------------------------------------------------------------------------------------------------- std::vector zen::extractSyncCfg(const MainConfiguration& mainCfg) { //merge first and additional pairs std::vector allPairs; allPairs.push_back(mainCfg.firstPair); allPairs.insert(allPairs.end(), mainCfg.additionalPairs.begin(), //add additional pairs mainCfg.additionalPairs.end()); std::vector output; //process all pairs for (auto i = allPairs.begin(); i != allPairs.end(); ++i) { SyncConfig syncCfg = i->altSyncConfig.get() ? *i->altSyncConfig : mainCfg.syncCfg; output.push_back( FolderPairSyncCfg(syncCfg.directionCfg.var == DirectionConfig::AUTOMATIC, syncCfg.handleDeletion, getFormattedDirectoryName(syncCfg.versioningDirectory), syncCfg.versionCountLimit)); } return output; } //------------------------------------------------------------------------------------------------------------ //test if user accidentally selected the wrong folders to sync bool significantDifferenceDetected(const SyncStatistics& folderPairStat) { //initial file copying shall not be detected as major difference if ((folderPairStat.getCreate() == 0 || folderPairStat.getCreate() == 0) && folderPairStat.getUpdate () == 0 && folderPairStat.getDelete () == 0 && folderPairStat.getConflict() == 0) return false; const int nonMatchingRows = folderPairStat.getCreate() + //folderPairStat.getUpdate() + -> not relevant when testing for "wrong folder selected" folderPairStat.getDelete () + folderPairStat.getConflict(); //? return nonMatchingRows >= 10 && nonMatchingRows > 0.5 * folderPairStat.getRowCount(); } //################################################################################################################# /* class PhysicalStatistics //counts *physical* operations, actual items processed (NOT disk accesses) and bytes transferred { public: PhysicalStatistics(const FolderComparison& folderCmp) : items(0) { delType =; std::for_each(begin(folderCmp), end(folderCmp), [&](const BaseDirMapping& baseMap) { recurse(baseMap); }); } int getItems() const { return items; } Int64 getBytes() const { return bytes; } private: enum DeletionType { DEL_PERMANENTLY, DEL_RECYCLE_BIN, VERSIONING_SAME_VOL, VERSIONING_DIFF_VOL, }; void recurse(const HierarchyObject& hierObj) { std::for_each(hierObj.refSubDirs ().begin(), hierObj.refSubDirs ().end(), [&](const DirMapping& dirObj ) { calcStats(dirObj ); }); std::for_each(hierObj.refSubFiles().begin(), hierObj.refSubFiles().end(), [&](const FileMapping& fileObj) { calcStats(fileObj); }); std::for_each(hierObj.refSubLinks().begin(), hierObj.refSubLinks().end(), [&](const SymLinkMapping& linkObj) { calcStats(linkObj); }); } void calcStats(const FileMapping& fileObj) { switch (fileObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_CREATE_NEW_LEFT: items += 2; bytes += to(fileObj.getFileSize()); break; case SO_CREATE_NEW_RIGHT: ++items; bytes += to(fileObj.getFileSize()); break; case SO_DELETE_LEFT: switch (delType) { case DEL_INSTANTLY: ++items; break; case DEL_COPY_DELETE: break; } break; case SO_DELETE_RIGHT: ++items; break; case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: ++items; break; case SO_MOVE_LEFT_SOURCE: //ignore; already counted case SO_MOVE_RIGHT_SOURCE: // break; case SO_OVERWRITE_LEFT: //todo: delete items += 2; //read + write bytes += to(fileObj.getFileSize()); break; case SO_OVERWRITE_RIGHT: //todo: delete items += 2; //read + write bytes += to(fileObj.getFileSize()); break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: ++items; break; case SO_UNRESOLVED_CONFLICT: case SO_DO_NOTHING: case SO_EQUAL: break; } } void calcStats(const SymLinkMapping& linkObj) { } void calcStats(const DirMapping& dirObj) { //since we model physical stats, we recurse only if deletion variant is "permanently" or "user-defined + different volume", //else deletion is done as a single physical operation } int items; Int64 bytes; DeletionType delType; }; */ //-------------------------------------------------------------------------------------------------------------- class DeletionHandling //e.g. generate name of alternate deletion directory (unique for session AND folder pair) { public: DeletionHandling(DeletionPolicy handleDel, //nothrow! const Zstring& versioningDir, const TimeComp& timeStamp, int versionCountLimit, size_t folderIndex, const Zstring& baseDirPf, //with separator postfix ProcessCallback& procCallback); ~DeletionHandling() { try { tryCleanup(false); } //always (try to) clean up, even if synchronization is aborted! catch (...) {} /* do not allow user callback: - make sure this stays non-blocking! - avoid throwing user abort exception again, leading to incomplete clean-up! */ } //clean-up temporary directory (recycle bin optimization) void tryCleanup(bool allowUserCallback = true); //throw FileError -> call this in non-exceptional coding, i.e. somewhere after sync! void removeFile (const Zstring& relativeName); //throw FileError void removeFolder(const Zstring& relativeName) { removeFolderInt(relativeName, nullptr, nullptr); }; //throw FileError void removeFolderUpdateStatistics(const Zstring& relativeName, int objectsExpected, Int64 dataExpected) { removeFolderInt(relativeName, &objectsExpected, &dataExpected); }; //throw FileError //in contrast to "removeFolder()" this function will update statistics! const std::wstring& getTxtRemovingFile () const { return txtRemovingFile; } // const std::wstring& getTxtRemovingSymLink() const { return txtRemovingSymlink; } //buffered status texts const std::wstring& getTxtRemovingDir () const { return txtRemovingDirectory; } // //evaluate whether a deletion will actually free space within a volume bool deletionFreesSpace() const; #ifdef FFS_WIN bool recyclerFallbackOnDelete() const { return recFallbackDelPermantently; } #endif private: DeletionHandling(const DeletionHandling&); DeletionHandling& operator=(const DeletionHandling&); void removeFolderInt(const Zstring& relativeName, const int* objectsExpected, const Int64* dataExpected); //throw FileError void setDeletionPolicy(DeletionPolicy newPolicy); FileVersioner& getOrCreateVersioner() //throw FileError! => dont create in DeletionHandling()!!! { if (!versioner.get()) versioner = make_unique(versioningDir_, timeStamp_, versionCountLimit_); //throw FileError return *versioner; }; ProcessCallback& procCallback_; const Zstring baseDirPf_; //ends with path separator const Zstring versioningDir_; const TimeComp timeStamp_; const int versionCountLimit_; #ifdef FFS_WIN Zstring recyclerTmpDirPf; //temporary folder holding files/folders for *deferred* recycling (postfixed with file name separator) std::vector toBeRecycled; //full path of files located in temporary folder, waiting to be recycled bool recFallbackDelPermantently; #endif //magage three states: allow dynamic fallback from recycler to permanent deletion DeletionPolicy deletionPolicy_; std::unique_ptr versioner; //used for DELETE_TO_VERSIONING; throw FileError in constructor => create on demand! //buffer status texts: std::wstring txtRemovingFile; std::wstring txtRemovingSymlink; std::wstring txtRemovingDirectory; bool cleanedUp; }; DeletionHandling::DeletionHandling(DeletionPolicy handleDel, //nothrow! const Zstring& versioningDir, const TimeComp& timeStamp, int versionCountLimit, size_t folderIndex, const Zstring& baseDirPf, //with separator postfix ProcessCallback& procCallback) : procCallback_(procCallback), baseDirPf_(baseDirPf), versioningDir_(versioningDir), timeStamp_(timeStamp), versionCountLimit_(versionCountLimit), #ifdef FFS_WIN recFallbackDelPermantently(false), #endif deletionPolicy_(DELETE_TO_RECYCLER), cleanedUp(false) { #ifdef FFS_WIN if (!baseDirPf.empty()) if (handleDel == DELETE_TO_RECYCLER && recycleBinStatus(baseDirPf) != STATUS_REC_EXISTS) { handleDel = DELETE_PERMANENTLY; //Windows' ::SHFileOperation() will do this anyway, but we have a better and faster deletion routine (e.g. on networks) recFallbackDelPermantently = true; } //assemble temporary recycler bin directory if (!baseDirPf_.empty()) { Zstring tempDir = baseDirPf_ + Zstr("FFS ") + formatTime(Zstr("%Y-%m-%d %H%M%S"), timeStamp); if (folderIndex > 0) tempDir += Zstr(" [") + numberTo(folderIndex + 1) + Zstr("]"); //e.g. C:\Source\FFS 2012-05-15 131513 [2] recyclerTmpDirPf = appendSeparator(tempDir); } #endif setDeletionPolicy(handleDel); } void DeletionHandling::setDeletionPolicy(DeletionPolicy newPolicy) { deletionPolicy_ = newPolicy; switch (deletionPolicy_) { case DELETE_PERMANENTLY: txtRemovingFile = _("Deleting file %x" ); txtRemovingDirectory = _("Deleting folder %x" ); txtRemovingSymlink = _("Deleting symbolic link %x"); break; case DELETE_TO_RECYCLER: txtRemovingFile = _("Moving file %x to recycle bin" ); txtRemovingDirectory = _("Moving folder %x to recycle bin" ); txtRemovingSymlink = _("Moving symbolic link %x to recycle bin"); break; case DELETE_TO_VERSIONING: txtRemovingFile = replaceCpy(_("Moving file %x to %y" ), L"%y", fmtFileName(versioningDir_)); txtRemovingDirectory = replaceCpy(_("Moving folder %x to %y" ), L"%y", fmtFileName(versioningDir_)); txtRemovingSymlink = replaceCpy(_("Moving symbolic link %x to %y"), L"%y", fmtFileName(versioningDir_)); break; } } #ifdef FFS_WIN namespace { class CallbackMassRecycling : public CallbackRecycling { public: CallbackMassRecycling(ProcessCallback& statusHandler) : statusHandler_(statusHandler), txtRecyclingFile(_("Moving file %x to recycle bin")) {} //may throw: first exception is swallowed, updateStatus() is then called again where it should throw again and the exception will propagate as expected virtual void updateStatus(const Zstring& currentItem) { if (!currentItem.empty()) statusHandler_.reportStatus(replaceCpy(txtRecyclingFile, L"%x", fmtFileName(currentItem))); //throw ? else statusHandler_.requestUiRefresh(); //throw ? } private: ProcessCallback& statusHandler_; const std::wstring txtRecyclingFile; }; } #endif void DeletionHandling::tryCleanup(bool allowUserCallback) //throw FileError { if (!cleanedUp) { switch (deletionPolicy_) { case DELETE_PERMANENTLY: break; case DELETE_TO_RECYCLER: #ifdef FFS_WIN if (!recyclerTmpDirPf.empty()) { //move content of temporary directory to recycle bin in a single call CallbackMassRecycling cbmr(procCallback_); recycleOrDelete(toBeRecycled, allowUserCallback ? &cbmr : nullptr); //throw FileError //clean up temp directory itself (should contain remnant empty directories only) removeDirectory(beforeLast(recyclerTmpDirPf, FILE_NAME_SEPARATOR)); //throw FileError } #endif break; case DELETE_TO_VERSIONING: if (versioner.get()) { if (allowUserCallback) { procCallback_.reportStatus(_("Removing old versions...")); //throw ? versioner->limitVersions([&] { procCallback_.requestUiRefresh(); /*throw ? */ }); //throw FileError } else versioner->limitVersions([] {}); //throw FileError } break; } cleanedUp = true; } } namespace { class CallbackMoveFileImpl : public CallbackMoveFile { public: CallbackMoveFileImpl(ProcessCallback& statusHandler, int* objectsReported) : statusHandler_ (statusHandler), objectsReported_(objectsReported), txtMovingFile (_("Moving file %x to %y")), txtMovingFolder (_("Moving folder %x to %y")) {} private: virtual void onBeforeFileMove(const Zstring& fileFrom, const Zstring& fileTo) { reportStatus(txtMovingFile, fileFrom, fileTo); } virtual void onBeforeDirMove (const Zstring& dirFrom, const Zstring& dirTo ) { reportStatus(txtMovingFolder, dirFrom, dirTo); } virtual void objectProcessed() //one call after each processed move { if (objectsReported_) { statusHandler_.updateProcessedData(1, 0); ++*objectsReported_; } } virtual void updateStatus(Int64 bytesDelta) { //statusHandler_.updateProcessedData(0, bytesDelta); //bytesReported_ += bytesDelta; -> statistics model *logical* operations! as such a file delete is only (1 obj/0 bytes)! Doesn't matter if it's actually copy + delete statusHandler_.requestUiRefresh(); } void reportStatus(const std::wstring& statusText, const Zstring& fileFrom, const Zstring& fileTo) const { statusHandler_.reportStatus(replaceCpy(replaceCpy(statusText, L"%x", fmtFileName(fileFrom)), L"%y", fmtFileName(fileTo))); }; ProcessCallback& statusHandler_; int* objectsReported_; //optional const std::wstring txtMovingFile; const std::wstring txtMovingFolder; }; struct CallbackRemoveDirImpl : public CallbackRemoveDir { CallbackRemoveDirImpl(ProcessCallback& statusHandler, const DeletionHandling& delHandling, int* objectsReported) : statusHandler_(statusHandler), delHandling_(delHandling), objectsReported_(objectsReported) {} private: virtual void notifyFileDeletion(const Zstring& filename) { processSingleObject(delHandling_.getTxtRemovingFile(), filename); } virtual void notifyDirDeletion (const Zstring& dirname ) { processSingleObject(delHandling_.getTxtRemovingDir (), dirname ); } void processSingleObject(const std::wstring& statusText, const Zstring& objName) { statusHandler_.reportStatus(replaceCpy(statusText, L"%x", fmtFileName(objName))); if (objectsReported_) { statusHandler_.updateProcessedData(1, 0); ++*objectsReported_; } } ProcessCallback& statusHandler_; const DeletionHandling& delHandling_; int* objectsReported_; //optional }; } void DeletionHandling::removeFile(const Zstring& relativeName) { const Zstring fullName = baseDirPf_ + relativeName; switch (deletionPolicy_) { case DELETE_PERMANENTLY: zen::removeFile(fullName); //[!] scope specifier resolves nameclash! break; case DELETE_TO_RECYCLER: #ifdef FFS_WIN { const Zstring targetFile = recyclerTmpDirPf + relativeName; //ends with path separator auto moveToTempDir = [&] { //performance optimization: Instead of moving each object into recycle bin separately, //we rename them one by one into a temporary directory and batch-recycle this directory after sync renameFile(fullName, targetFile); //throw FileError toBeRecycled.push_back(targetFile); }; try { moveToTempDir(); //throw FileError } catch (FileError&) { if (somethingExists(fullName)) { const Zstring targetDir = beforeLast(targetFile, FILE_NAME_SEPARATOR); if (!dirExists(targetDir)) { makeDirectory(targetDir); //throw FileError -> may legitimately fail on Linux if permissions are missing moveToTempDir(); //throw FileError -> this should work now! } else throw; } } } #elif defined FFS_LINUX recycleOrDelete(fullName); //throw FileError #endif break; case DELETE_TO_VERSIONING: { CallbackMoveFileImpl callback(procCallback_, nullptr); //we do *not* report statistics in this method getOrCreateVersioner().revisionFile(fullName, relativeName, callback); //throw FileError } break; } } void DeletionHandling::removeFolderInt(const Zstring& relativeName, const int* objectsExpected, const Int64* dataExpected) //throw FileError { const Zstring fullName = baseDirPf_ + relativeName; int objectsReported = 0; //use *only* if "objectsExpected" is bound! //in error situation: undo communication of processed amount of data ScopeGuard guardStatistics = makeGuard([&] { procCallback_.updateProcessedData(-objectsReported, 0); }); switch (deletionPolicy_) { case DELETE_PERMANENTLY: { CallbackRemoveDirImpl remDirCallback(procCallback_, *this, objectsExpected ? &objectsReported : nullptr); removeDirectory(fullName, &remDirCallback); } break; case DELETE_TO_RECYCLER: #ifdef FFS_WIN { const Zstring targetDir = recyclerTmpDirPf + relativeName; auto moveToTempDir = [&] { //performance optimization: Instead of moving each object into recycle bin separately, //we rename them one by one into a temporary directory and batch-recycle this directory after sync renameFile(fullName, targetDir); //throw FileError toBeRecycled.push_back(targetDir); }; try { moveToTempDir(); //throw FileError } catch (FileError&) { if (somethingExists(fullName)) { const Zstring targetSuperDir = beforeLast(targetDir, FILE_NAME_SEPARATOR); if (!dirExists(targetSuperDir)) { makeDirectory(targetSuperDir); //throw FileError -> may legitimately fail on Linux if permissions are missing moveToTempDir(); //throw FileError -> this should work now! } else throw; } } } #elif defined FFS_LINUX recycleOrDelete(fullName); //throw FileError #endif if (objectsExpected) //even though we have only one disk access, we completed "objectsExpected" logical operations! { procCallback_.updateProcessedData(*objectsExpected, 0); objectsReported += *objectsExpected; } break; case DELETE_TO_VERSIONING: { CallbackMoveFileImpl callback(procCallback_, objectsExpected ? &objectsReported : nullptr); getOrCreateVersioner().revisionDir(fullName, relativeName, callback); //throw FileError } break; } //inform about the (remaining) processed amount of data if (objectsExpected && dataExpected) { guardStatistics.dismiss(); if (*objectsExpected != objectsReported || *dataExpected != 0) //adjust total: may have changed after comparison! procCallback_.updateTotalData(objectsReported - *objectsExpected, -*dataExpected); } } //evaluate whether a deletion will actually free space within a volume bool DeletionHandling::deletionFreesSpace() const { switch (deletionPolicy_) { case DELETE_PERMANENTLY: return true; case DELETE_TO_RECYCLER: return false; //in general... (unless Recycle Bin is full) case DELETE_TO_VERSIONING: switch (zen::onSameVolume(baseDirPf_, versioningDir_)) { case IS_SAME_YES: return false; case IS_SAME_NO: return true; //but other volume (versioningDir) may become full... case IS_SAME_CANT_SAY: return true; //a rough guess! } } assert(false); return true; } //------------------------------------------------------------------------------------------------------------ namespace { class DiskSpaceNeeded { public: static std::pair calculate(const BaseDirMapping& baseObj, bool freeSpaceDelLeft, bool freeSpaceDelRight) { DiskSpaceNeeded inst(baseObj, freeSpaceDelLeft, freeSpaceDelRight); return std::make_pair(inst.spaceNeededLeft, inst.spaceNeededRight); } private: DiskSpaceNeeded(const BaseDirMapping& baseObj, bool freeSpaceDelLeft, bool freeSpaceDelRight) : freeSpaceDelLeft_(freeSpaceDelLeft), freeSpaceDelRight_(freeSpaceDelRight) { recurse(baseObj); } void recurse(const HierarchyObject& hierObj) { //don't process directories //process files for (auto i = hierObj.refSubFiles().begin(); i != hierObj.refSubFiles().end(); ++i) switch (i->getSyncOperation()) //evaluate comparison result and sync direction { case SO_CREATE_NEW_LEFT: spaceNeededLeft += to(i->getFileSize()); break; case SO_CREATE_NEW_RIGHT: spaceNeededRight += to(i->getFileSize()); break; case SO_DELETE_LEFT: if (freeSpaceDelLeft_) spaceNeededLeft -= to(i->getFileSize()); break; case SO_DELETE_RIGHT: if (freeSpaceDelRight_) spaceNeededRight -= to(i->getFileSize()); break; case SO_OVERWRITE_LEFT: if (freeSpaceDelLeft_) spaceNeededLeft -= to(i->getFileSize()); spaceNeededLeft += to(i->getFileSize()); break; case SO_OVERWRITE_RIGHT: if (freeSpaceDelRight_) spaceNeededRight -= to(i->getFileSize()); spaceNeededRight += to(i->getFileSize()); break; case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: break; } //symbolic links //[...] //recurse into sub-dirs std::for_each(hierObj.refSubDirs().begin(), hierObj.refSubDirs().end(), [&](const HierarchyObject& subDir) { this->recurse(subDir); }); //for (auto& subDir : hierObj.refSubDirs()) // recurse(subDir); } const bool freeSpaceDelLeft_; const bool freeSpaceDelRight_; Int64 spaceNeededLeft; Int64 spaceNeededRight; }; //---------------------------------------------------------------------------------------- class SynchronizeFolderPair { public: SynchronizeFolderPair(ProcessCallback& procCallback, bool verifyCopiedFiles, bool copyFilePermissions, bool transactionalFileCopy, #ifdef FFS_WIN shadow::ShadowCopy* shadowCopyHandler, #endif DeletionHandling& delHandlingLeft, DeletionHandling& delHandlingRight) : procCallback_(procCallback), #ifdef FFS_WIN shadowCopyHandler_(shadowCopyHandler), #endif delHandlingLeft_(delHandlingLeft), delHandlingRight_(delHandlingRight), verifyCopiedFiles_(verifyCopiedFiles), copyFilePermissions_(copyFilePermissions), transactionalFileCopy_(transactionalFileCopy), txtCreatingFile (_("Creating file %x" )), txtCreatingLink (_("Creating symbolic link %x" )), txtCreatingFolder (_("Creating folder %x" )), txtOverwritingFile (_("Overwriting file %x" )), txtOverwritingLink (_("Overwriting symbolic link %x")), txtVerifying (_("Verifying file %x" )), txtWritingAttributes(_("Updating attributes of %x" )), txtMovingFile (_("Moving file %x to %y")) {} void startSync(BaseDirMapping& baseMap) { runZeroPass(baseMap); //first process file moves runPass(baseMap); //delete files (or overwrite big ones with smaller ones) runPass(baseMap); //copy rest } private: enum PassId { PASS_ONE, //delete files PASS_TWO, //create, modify PASS_NEVER //skip }; static PassId getPass(const FileMapping& fileObj); static PassId getPass(const SymLinkMapping& linkObj); static PassId getPass(const DirMapping& dirObj); template void prepare2StepMove(FileMapping& sourceObj, FileMapping& targetObj); //throw FileError bool createParentDir(FileSystemObject& fsObj); //throw FileError template void manageFileMove(FileMapping& sourceObj, FileMapping& targetObj); //throw FileError void runZeroPass(HierarchyObject& hierObj); template void runPass(HierarchyObject& hierObj); void synchronizeFile(FileMapping& fileObj); template void synchronizeFileInt(FileMapping& fileObj, SyncOperation syncOp); void synchronizeLink(SymLinkMapping& linkObj); template void synchronizeLinkInt(SymLinkMapping& linkObj, SyncOperation syncOp); void synchronizeFolder(DirMapping& dirObj); template void synchronizeFolderInt(DirMapping& dirObj, SyncOperation syncOp); void reportInfo (const std::wstring& rawText, const Zstring& objname) const { procCallback_.reportInfo (replaceCpy(rawText, L"%x", fmtFileName(objname))); }; void reportStatus(const std::wstring& rawText, const Zstring& objname) const { procCallback_.reportStatus(replaceCpy(rawText, L"%x", fmtFileName(objname))); }; //more low level helper template void copyFileUpdatingTo(const FileMapping& fileObj, const DelTargetCommand& cmd, FileAttrib& newAttr) const; void verifyFileCopy(const Zstring& source, const Zstring& target) const; template DeletionHandling& getDelHandling(); ProcessCallback& procCallback_; #ifdef FFS_WIN shadow::ShadowCopy* shadowCopyHandler_; //optional! #endif DeletionHandling& delHandlingLeft_; DeletionHandling& delHandlingRight_; const bool verifyCopiedFiles_; const bool copyFilePermissions_; const bool transactionalFileCopy_; //preload status texts const std::wstring txtCreatingFile; const std::wstring txtCreatingLink; const std::wstring txtCreatingFolder; const std::wstring txtOverwritingFile; const std::wstring txtOverwritingLink; const std::wstring txtVerifying; const std::wstring txtWritingAttributes; const std::wstring txtMovingFile; }; //--------------------------------------------------------------------------------------------------------------- template <> inline DeletionHandling& SynchronizeFolderPair::getDelHandling() { return delHandlingLeft_; } template <> inline DeletionHandling& SynchronizeFolderPair::getDelHandling() { return delHandlingRight_; } } /* __________________________ |Move algorithm, 0th pass| -------------------------- 1. loop over hierarchy and find "move source" 2. check whether parent directory of "move source" is going to be deleted or location of "move source" may lead to name clash with other dir/symlink -> no: delay move until 2nd pass 3. create move target's parent directory recursively + execute move do we have name clash? -> prepare a 2-step move operation: 1. move source to root and update "move target" accordingly 2. delay move until 2nd pass 4. If any of the operations above did not succeed (even after retry), update statistics and revert to "copy + delete" Note: first pass may delete "move source"!!! __________________ |killer-scenarios| ------------------ propagate the following move sequences: I) a -> a/a caveat sync'ing parent directory first leads to circular dependency! II) a/a -> a caveat: fixing name clash will remove source! III) c -> d caveat: move-sequence needs to be processed in correct order! b -> c/b a -> b/a */ namespace { template inline bool haveNameClash(const Zstring& shortname, Mapping& m) { return std::any_of(m.begin(), m.end(), [&](const typename Mapping::value_type& obj) { return EqualFilename()(obj.getObjShortName(), shortname); }); } Zstring findUnusedTempName(const Zstring& filename) { Zstring output = filename + zen::TEMP_FILE_ENDING; //ensure uniqueness for (int i = 1; somethingExists(output); ++i) output = filename + Zchar('_') + numberTo(i) + zen::TEMP_FILE_ENDING; return output; } } template void SynchronizeFolderPair::prepare2StepMove(FileMapping& sourceObj, FileMapping& targetObj) //throw FileError { const Zstring& source = sourceObj.getFullName(); const Zstring& tmpTarget = findUnusedTempName(sourceObj.getBaseDirPf() + sourceObj.getShortName()); //this could still lead to a name-clash in obscure cases, if some file exists on the other side with //the very same (.ffs_tmp) name and is copied before the second step of the move is executed //good news: even in this pathologic case, this may only prevent the copy of the other file, but not the move reportInfo(replaceCpy(txtMovingFile, L"%y", fmtFileName(tmpTarget)), source); renameFile(source, tmpTarget); //throw FileError //update file hierarchy const FileDescriptor descrSource(sourceObj.getLastWriteTime(), sourceObj.getFileSize (), sourceObj.getFileId ()); sourceObj.removeObject(); FileMapping& tempFile = sourceObj.root().addSubFile(afterLast(tmpTarget, FILE_NAME_SEPARATOR), descrSource); //prepare move in second pass tempFile.setSyncDir(side == LEFT_SIDE ? SYNC_DIR_LEFT : SYNC_DIR_RIGHT); targetObj.setMoveRef(tempFile .getId()); tempFile .setMoveRef(targetObj.getId()); //NO statistics update! procCallback_.requestUiRefresh(); //may throw } bool SynchronizeFolderPair::createParentDir(FileSystemObject& fsObj) //throw FileError, "false" on name clash { if (DirMapping* parentDir = dynamic_cast(&fsObj.parent())) { if (!createParentDir(*parentDir)) return false; //detect (and try to resolve) file type conflicts: 1. symlinks 2. files const Zstring& shortname = parentDir->getObjShortName(); if (haveNameClash(shortname, parentDir->parent().refSubLinks()) || haveNameClash(shortname, parentDir->parent().refSubFiles())) return false; //in this context "parentDir" cannot be scheduled for deletion since it contains a "move target"! //note: if parentDir were deleted, we'd end up destroying "fsObj"! assert(parentDir->getSyncOperation() != SO_DELETE_LEFT && parentDir->getSyncOperation() != SO_DELETE_RIGHT); synchronizeFolder(*parentDir); //throw FileError } return true; } template void SynchronizeFolderPair::manageFileMove(FileMapping& sourceObj, FileMapping& targetObj) //throw FileError { assert((sourceObj.getSyncOperation() == SO_MOVE_LEFT_SOURCE && targetObj.getSyncOperation() == SO_MOVE_LEFT_TARGET && side == LEFT_SIDE) || (sourceObj.getSyncOperation() == SO_MOVE_RIGHT_SOURCE && targetObj.getSyncOperation() == SO_MOVE_RIGHT_TARGET && side == RIGHT_SIDE)); const bool sourceWillBeDeleted = [&]() -> bool { if (DirMapping* parentDir = dynamic_cast(&sourceObj.parent())) { switch (parentDir->getSyncOperation()) //evaluate comparison result and sync direction { case SO_DELETE_LEFT: case SO_DELETE_RIGHT: return true; //we need to do something about it case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: break; } } return false; }(); auto haveNameClash = [](const FileMapping& fileObj) { return ::haveNameClash(fileObj.getObjShortName(), fileObj.parent().refSubLinks()) || ::haveNameClash(fileObj.getObjShortName(), fileObj.parent().refSubDirs()); }; if (sourceWillBeDeleted || haveNameClash(sourceObj)) { //prepare for move now: - revert to 2-step move on name clashes if (haveNameClash(targetObj) || !createParentDir(targetObj)) //throw FileError return prepare2StepMove(sourceObj, targetObj); //throw FileError //finally start move! this should work now: synchronizeFile(sourceObj); //throw FileError } //else: sourceObj will not be deleted, and is not standing in the way => delay to second pass //note: this case may include "move sources" from two-step sub-routine! } void SynchronizeFolderPair::runZeroPass(HierarchyObject& hierObj) { //search for file move-operations for (auto iter = hierObj.refSubFiles().begin(); iter != hierObj.refSubFiles().end(); ++iter) //VS 2010 crashes if we use for_each + lambda here... { FileMapping& fileObj = *iter; const SyncOperation syncOp = fileObj.getSyncOperation(); switch (syncOp) //evaluate comparison result and sync direction { case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: { FileMapping* sourceObj = &fileObj; if (FileMapping* targetObj = dynamic_cast(FileSystemObject::retrieve(fileObj.getMoveRef()))) { if (!tryReportingError([&] { if (syncOp == SO_MOVE_LEFT_SOURCE) this->manageFileMove(*sourceObj, *targetObj); else this->manageFileMove(*sourceObj, *targetObj); }, procCallback_)) { //move operation has failed! We cannot allow to continue and have move source's parent directory deleted, messing up statistics! // => revert to ordinary "copy + delete" auto getStat = [&]() -> std::pair { SyncStatistics statSrc(*sourceObj); SyncStatistics statTrg(*targetObj); return std::make_pair(getCUD(statSrc) + getCUD(statTrg), statSrc.getDataToProcess() + statTrg.getDataToProcess()); }; const auto statBefore = getStat(); sourceObj->setMoveRef(nullptr); targetObj->setMoveRef(nullptr); const auto statAfter = getStat(); //fix statistics to total to match "copy + delete" procCallback_.updateTotalData(statAfter.first - statBefore.first, statAfter.second - statBefore.second); } } } break; case SO_MOVE_LEFT_TARGET: //it's enough to try each move-pair *once* case SO_MOVE_RIGHT_TARGET: // case SO_DELETE_LEFT: case SO_DELETE_RIGHT: case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: break; } } std::for_each(hierObj.refSubDirs().begin(), hierObj.refSubDirs().end(), [&](DirMapping& dirObj) { this->runZeroPass(dirObj); /*recurse */ }); } //--------------------------------------------------------------------------------------------------------------- //1st, 2nd pass requirements: // - avoid disk space shortage: 1. delete files, 2. overwrite big with small files first // - support change in type: overwrite file by directory, symlink by file, ect. inline SynchronizeFolderPair::PassId SynchronizeFolderPair::getPass(const FileMapping& fileObj) { switch (fileObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_DELETE_LEFT: case SO_DELETE_RIGHT: return PASS_ONE; case SO_OVERWRITE_LEFT: return fileObj.getFileSize() > fileObj.getFileSize() ? PASS_ONE : PASS_TWO; case SO_OVERWRITE_RIGHT: return fileObj.getFileSize() < fileObj.getFileSize() ? PASS_ONE : PASS_TWO; case SO_MOVE_LEFT_SOURCE: // case SO_MOVE_RIGHT_SOURCE: // [!] return PASS_NEVER; case SO_MOVE_LEFT_TARGET: // case SO_MOVE_RIGHT_TARGET: //to be processed in second pass, after "move target" parent directory was created! return PASS_TWO; case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: return PASS_TWO; } assert(false); return PASS_TWO; //dummy } inline SynchronizeFolderPair::PassId SynchronizeFolderPair::getPass(const SymLinkMapping& linkObj) { switch (linkObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_DELETE_LEFT: case SO_DELETE_RIGHT: return PASS_ONE; //make sure to delete symlinks in first pass, and equally named file or dir in second pass: usecase "overwrite symlink with regular file"! case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: return PASS_TWO; } assert(false); return PASS_TWO; //dummy } inline SynchronizeFolderPair::PassId SynchronizeFolderPair::getPass(const DirMapping& dirObj) { switch (dirObj.getSyncOperation()) //evaluate comparison result and sync direction { case SO_DELETE_LEFT: case SO_DELETE_RIGHT: return PASS_ONE; case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: return PASS_TWO; } assert(false); return PASS_TWO; //dummy } template void SynchronizeFolderPair::runPass(HierarchyObject& hierObj) { //synchronize files: std::for_each(hierObj.refSubFiles().begin(), hierObj.refSubFiles().end(), [&](FileMapping& fileObj) { if (pass == this->getPass(fileObj)) //"this->" required by two-pass lookup as enforced by GCC 4.7 tryReportingError([&] { synchronizeFile(fileObj); }, procCallback_); }); //synchronize symbolic links: std::for_each(hierObj.refSubLinks().begin(), hierObj.refSubLinks().end(), [&](SymLinkMapping& linkObj) { if (pass == this->getPass(linkObj)) tryReportingError([&] { synchronizeLink(linkObj); }, procCallback_); }); //synchronize folders: std::for_each(hierObj.refSubDirs().begin(), hierObj.refSubDirs().end(), [&](DirMapping& dirObj) { if (pass == this->getPass(dirObj)) tryReportingError([&] { synchronizeFolder(dirObj); }, procCallback_); this->runPass(dirObj); //recurse }); } //--------------------------------------------------------------------------------------------------------------- namespace { inline bool getTargetDirection(SyncOperation syncOp, SelectedSide* side) { switch (syncOp) { case SO_CREATE_NEW_LEFT: case SO_DELETE_LEFT: case SO_OVERWRITE_LEFT: case SO_COPY_METADATA_TO_LEFT: case SO_MOVE_LEFT_SOURCE: case SO_MOVE_LEFT_TARGET: *side = LEFT_SIDE; return true; case SO_CREATE_NEW_RIGHT: case SO_DELETE_RIGHT: case SO_OVERWRITE_RIGHT: case SO_COPY_METADATA_TO_RIGHT: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_RIGHT_TARGET: *side = RIGHT_SIDE; return true; case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: break; //nothing to do } return false; } } inline void SynchronizeFolderPair::synchronizeFile(FileMapping& fileObj) { const SyncOperation syncOp = fileObj.getSyncOperation(); SelectedSide sideTrg = LEFT_SIDE; if (getTargetDirection(syncOp, &sideTrg)) { if (sideTrg == LEFT_SIDE) synchronizeFileInt(fileObj, syncOp); else synchronizeFileInt(fileObj, syncOp); } } template void SynchronizeFolderPair::synchronizeFileInt(FileMapping& fileObj, SyncOperation syncOp) { static const SelectedSide sideSrc = OtherSide::result; switch (syncOp) { case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: { const Zstring& target = fileObj.getBaseDirPf() + fileObj.getRelativeName(); //can't use "getFullName" as target is not yet existing reportInfo(txtCreatingFile, target); try { FileAttrib newAttr; copyFileUpdatingTo(fileObj, [] {}, /*no target to delete*/ newAttr); //throw FileError procCallback_.updateProcessedData(1, 0); //processed data is communicated in copyFileUpdatingTo()! const FileDescriptor descrSource(newAttr.modificationTime, newAttr.fileSize, newAttr.sourceFileId); const FileDescriptor descrTarget(newAttr.modificationTime, newAttr.fileSize, newAttr.targetFileId); fileObj.syncTo(descrTarget, &descrSource); //update FileMapping } catch (FileError&) { if (somethingExists(fileObj.getFullName())) //do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it! throw; //source deleted meanwhile...nothing was done (logical point of view!) procCallback_.updateTotalData(-1, -to(fileObj.getFileSize())); fileObj.removeObject(); } } break; case SO_DELETE_LEFT: case SO_DELETE_RIGHT: reportInfo(getDelHandling().getTxtRemovingFile(), fileObj.getFullName()); getDelHandling().removeFile(fileObj.getObjRelativeName()); //throw FileError fileObj.removeObject(); //update FileMapping procCallback_.updateProcessedData(1, 0); break; case SO_MOVE_LEFT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_RIGHT_TARGET: if (FileMapping* targetObj = dynamic_cast(FileSystemObject::retrieve(fileObj.getMoveRef()))) { FileMapping* sourceObj = &fileObj; if (syncOp != SO_MOVE_LEFT_SOURCE && syncOp != SO_MOVE_RIGHT_SOURCE) std::swap(sourceObj, targetObj); assert((sourceObj->getSyncOperation() == SO_MOVE_LEFT_SOURCE && targetObj->getSyncOperation() == SO_MOVE_LEFT_TARGET && sideTrg == LEFT_SIDE) || (sourceObj->getSyncOperation() == SO_MOVE_RIGHT_SOURCE && targetObj->getSyncOperation() == SO_MOVE_RIGHT_TARGET && sideTrg == RIGHT_SIDE)); const Zstring& source = sourceObj->getFullName(); const Zstring& target = targetObj->getBaseDirPf() + targetObj->getRelativeName(); reportInfo(replaceCpy(txtMovingFile, L"%y", fmtFileName(target)), source); renameFile(source, target); //throw FileError const FileDescriptor descrTarget(sourceObj->getLastWriteTime(), sourceObj->getFileSize (), sourceObj->getFileId ()); sourceObj->removeObject(); // targetObj->syncTo(descrTarget); //update FileMapping procCallback_.updateProcessedData(1, 0); } break; case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: { const Zstring& target = fileObj.getBaseDirPf() + fileObj.getRelativeName(); //respect differences in case of source object reportInfo(txtOverwritingFile, target); FileAttrib newAttr; copyFileUpdatingTo(fileObj, [&] //delete target at appropriate time { reportStatus(this->getDelHandling().getTxtRemovingFile(), fileObj.getFullName()); this->getDelHandling().removeFile(fileObj.getObjRelativeName()); //throw FileError fileObj.removeObject(); //remove file from FileMapping, to keep in sync (if subsequent copying fails!!) reportStatus(txtOverwritingFile, target); //restore status text copy file }, newAttr); const FileDescriptor descrSource(newAttr.modificationTime, newAttr.fileSize, newAttr.sourceFileId); const FileDescriptor descrTarget(newAttr.modificationTime, newAttr.fileSize, newAttr.targetFileId); fileObj.syncTo(descrTarget, &descrSource); //update FileMapping procCallback_.updateProcessedData(1, 0); } break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: { reportInfo(txtWritingAttributes, fileObj.getFullName()); if (fileObj.getShortName() != fileObj.getShortName()) //adapt difference in case (windows only) renameFile(fileObj.getFullName(), beforeLast(fileObj.getFullName(), FILE_NAME_SEPARATOR) + FILE_NAME_SEPARATOR + fileObj.getShortName()); //throw FileError if (!sameFileTime(fileObj.getLastWriteTime(), fileObj.getLastWriteTime(), 2)) //respect 2 second FAT/FAT32 precision setFileTime(fileObj.getFullName(), fileObj.getLastWriteTime(), SYMLINK_FOLLOW); //throw FileError //do NOT read *current* source file time, but use buffered value which corresponds to time of comparison! const FileDescriptor descrTarget(fileObj.getLastWriteTime(), fileObj.getFileSize (), fileObj.getFileId ()); fileObj.syncTo(descrTarget); //-> both sides *should* be completely equal now... procCallback_.updateProcessedData(1, 0); } break; case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: return; //no update on processed data! } procCallback_.requestUiRefresh(); //may throw } inline void SynchronizeFolderPair::synchronizeLink(SymLinkMapping& linkObj) { const SyncOperation syncOp = linkObj.getSyncOperation(); SelectedSide sideTrg = LEFT_SIDE; if (getTargetDirection(syncOp, &sideTrg)) { if (sideTrg == LEFT_SIDE) synchronizeLinkInt(linkObj, syncOp); else synchronizeLinkInt(linkObj, syncOp); } } template void SynchronizeFolderPair::synchronizeLinkInt(SymLinkMapping& linkObj, SyncOperation syncOp) { static const SelectedSide sideSrc = OtherSide::result; auto deleteSymlink = [&] { switch (linkObj.getLinkType()) { case LinkDescriptor::TYPE_DIR: this->getDelHandling().removeFolder(linkObj.getObjRelativeName()); //throw FileError break; case LinkDescriptor::TYPE_FILE: //Windows: true file symlink; Linux: file-link or broken link this->getDelHandling().removeFile(linkObj.getObjRelativeName()); //throw FileError break; } }; switch (syncOp) { case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: { const Zstring& target = linkObj.getBaseDirPf() + linkObj.getRelativeName(); reportInfo(txtCreatingLink, target); try { zen::copySymlink(linkObj.getFullName(), target, copyFilePermissions_); //throw FileError procCallback_.updateProcessedData(1, 0); linkObj.copyTo(); //update SymLinkMapping } catch (FileError&) { if (somethingExists(linkObj.getFullName())) //do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it! throw; //source deleted meanwhile...nothing was done (logical point of view!) procCallback_.updateTotalData(-1, 0); linkObj.removeObject(); } } break; case SO_DELETE_LEFT: case SO_DELETE_RIGHT: reportInfo(getDelHandling().getTxtRemovingSymLink(), linkObj.getFullName()); deleteSymlink(); //throw FileError linkObj.removeObject(); //update SymLinkMapping procCallback_.updateProcessedData(1, 0); break; case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: { const Zstring& target = linkObj.getBaseDirPf() + linkObj.getRelativeName(); //respect differences in case of source object reportInfo(txtOverwritingLink, target); reportStatus(getDelHandling().getTxtRemovingSymLink(), linkObj.getFullName()); deleteSymlink(); //throw FileError linkObj.removeObject(); //remove file from FileMapping, to keep in sync (if subsequent copying fails!!) reportStatus(txtOverwritingLink, target); //restore status text zen::copySymlink(linkObj.getFullName(), target, copyFilePermissions_); //throw FileError linkObj.copyTo(); //update SymLinkMapping procCallback_.updateProcessedData(1, 0); } break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: reportInfo(txtWritingAttributes, linkObj.getFullName()); if (linkObj.getShortName() != linkObj.getShortName()) //adapt difference in case (windows only) renameFile(linkObj.getFullName(), beforeLast(linkObj.getFullName(), FILE_NAME_SEPARATOR) + FILE_NAME_SEPARATOR + linkObj.getShortName()); //throw FileError if (!sameFileTime(linkObj.getLastWriteTime(), linkObj.getLastWriteTime(), 2)) //respect 2 second FAT/FAT32 precision setFileTime(linkObj.getFullName(), linkObj.getLastWriteTime(), SYMLINK_DIRECT); //throw FileError linkObj.copyTo(); //-> both sides *should* be completely equal now... procCallback_.updateProcessedData(1, 0); break; case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_DO_NOTHING: case SO_EQUAL: case SO_UNRESOLVED_CONFLICT: return; //no update on processed data! } procCallback_.requestUiRefresh(); //may throw } inline void SynchronizeFolderPair::synchronizeFolder(DirMapping& dirObj) { const SyncOperation syncOp = dirObj.getSyncOperation(); SelectedSide sideTrg = LEFT_SIDE; if (getTargetDirection(syncOp, &sideTrg)) { if (sideTrg == LEFT_SIDE) synchronizeFolderInt(dirObj, syncOp); else synchronizeFolderInt(dirObj, syncOp); } } template void SynchronizeFolderPair::synchronizeFolderInt(DirMapping& dirObj, SyncOperation syncOp) { static const SelectedSide sideSrc = OtherSide::result; switch (syncOp) { case SO_CREATE_NEW_LEFT: case SO_CREATE_NEW_RIGHT: if (somethingExists(dirObj.getFullName())) //do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it! { const Zstring& target = dirObj.getBaseDirPf() + dirObj.getRelativeName(); reportInfo(txtCreatingFolder, target); try { makeNewDirectory(target, dirObj.getFullName(), copyFilePermissions_); //no symlink copying! } catch (const ErrorTargetExisting&) { if (!dirExists(target)) throw; } //clash with file (dir-symlink is okay) dirObj.copyTo(); //update DirMapping procCallback_.updateProcessedData(1, 0); } else //source deleted meanwhile...nothing was done (logical point of view!) -> uh....what about a temporary network drop??? { // throw FileError const SyncStatistics subStats(dirObj); procCallback_.updateTotalData(-getCUD(subStats) - 1, -subStats.getDataToProcess()); dirObj.refSubFiles().clear(); //...then remove sub-objects dirObj.refSubLinks().clear(); // dirObj.refSubDirs ().clear(); // dirObj.removeObject(); } break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: reportInfo(txtWritingAttributes, dirObj.getFullName()); if (dirObj.getShortName() != dirObj.getShortName()) //adapt difference in case (windows only) renameFile(dirObj.getFullName(), beforeLast(dirObj.getFullName(), FILE_NAME_SEPARATOR) + FILE_NAME_SEPARATOR + dirObj.getShortName()); //throw FileError //copyFileTimes -> useless at this time: modification time changes with each child-object creation/deletion dirObj.copyTo(); //-> both sides *should* be completely equal now... procCallback_.updateProcessedData(1, 0); break; case SO_DELETE_LEFT: case SO_DELETE_RIGHT: { reportInfo(getDelHandling().getTxtRemovingDir(), dirObj.getFullName()); const SyncStatistics subStats(dirObj); //counts sub-objects only! getDelHandling().removeFolderUpdateStatistics(dirObj.getObjRelativeName(), 1 + getCUD(subStats), subStats.getDataToProcess()); //throw FileError //this call covers progress indicator updates for dir + sub-objects! dirObj.refSubFiles().clear(); //...then remove everything dirObj.refSubLinks().clear(); // dirObj.refSubDirs ().clear(); // dirObj.removeObject(); // } break; case SO_OVERWRITE_RIGHT: case SO_OVERWRITE_LEFT: case SO_MOVE_LEFT_SOURCE: case SO_MOVE_RIGHT_SOURCE: case SO_MOVE_LEFT_TARGET: case SO_MOVE_RIGHT_TARGET: assert(false); case SO_UNRESOLVED_CONFLICT: case SO_DO_NOTHING: case SO_EQUAL: return; //no update on processed data! } procCallback_.requestUiRefresh(); //may throw } namespace { void makeSameLength(std::wstring& first, std::wstring& second) { const size_t maxPref = std::max(first.length(), second.length()); first .resize(maxPref, L' '); second.resize(maxPref, L' '); } /* struct LessDependentDirectory : public std::binary_function { -> a *very* bad idea: this is NOT a strict weak ordering! No transitivity of equivalence! bool operator()(const Zstring& lhs, const Zstring& rhs) const { return LessFilename()(Zstring(lhs.c_str(), std::min(lhs.length(), rhs.length())), Zstring(rhs.c_str(), std::min(lhs.length(), rhs.length()))); } }; */ template //create base directories first (if not yet existing) -> no symlink or attribute copying! bool createBaseDirectory(BaseDirMapping& baseMap, ProcessCallback& callback) //nothrow; return false if fatal error occurred { const Zstring dirname = beforeLast(baseMap.getBaseDirPf(), FILE_NAME_SEPARATOR); if (!dirname.empty()) { if (baseMap.isExisting()) //atomicity: do NOT check directory existence again! { //just convenience: exit sync right here instead of showing tons of error messages during file copy return tryReportingError([&] { if (!dirExistsUpdating(dirname, false, callback)) throw FileError(replaceCpy(_("Cannot find folder %x."), L"%x", fmtFileName(dirname))); //this should really be a "fatal error" if not recoverable }, callback); //may throw in error-callback! } else //create target directory: user presumably ignored error "dir existing" in order to have it created automatically { bool temporaryNetworkDrop = false; bool rv = tryReportingError([&] { try { makeNewDirectory(dirname, Zstring(), false); //FileError, ErrorTargetExisting //a nice race-free check and set operation! baseMap.setExisting(true); //update our model! } catch (const ErrorTargetExisting&) { //TEMPORARY network drop: base directory not found during comparison, but reappears during synchronization //=> sync-directions are based on false assumptions! Abort. callback.reportFatalError(replaceCpy(_("Target folder %x already existing."), L"%x", fmtFileName(dirname))); temporaryNetworkDrop = true; //Is it possible we're catching a "false-positive" here, could FFS have created the directory indirectly after comparison? // 1. deletion handling: recycler -> no, temp directory created only at first deletion // 2. deletion handling: versioning -> " // 3. log file creates containing folder -> no, log only created in batch mode, and only *before* comparison } }, callback); //may throw in error-callback! return rv && !temporaryNetworkDrop; } } return true; } } void zen::synchronize(const TimeComp& timeStamp, xmlAccess::OptionalDialogs& warnings, bool verifyCopiedFiles, bool copyLockedFiles, bool copyFilePermissions, bool transactionalFileCopy, bool runWithBackgroundPriority, const std::vector& syncConfig, FolderComparison& folderCmp, ProcessCallback& callback) { //specify process and resource handling priorities std::unique_ptr backgroundPrio; if (runWithBackgroundPriority) backgroundPrio.reset(new ScheduleForBackgroundProcessing); //prevent operating system going into sleep state PreventStandby dummy; (void)dummy; //PERF_START; if (syncConfig.size() != folderCmp.size()) throw std::logic_error("Programming Error: Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); //inform about the total amount of data that will be processed from now on const SyncStatistics statisticsTotal(folderCmp); //keep at beginning so that all gui elements are initialized properly callback.initNewPhase(getCUD(statisticsTotal), statisticsTotal.getDataToProcess(), ProcessCallback::PHASE_SYNCHRONIZING); std::deque skipFolderPair(folderCmp.size()); //folder pairs may be skipped after fatal errors were found //initialize deletion handling: already required when checking for warnings FixedList delHandlerL; //we can't use a FixedList> because DeletionHandling is not copy-constructable FixedList delHandlerR; for (auto j = begin(folderCmp); j != end(folderCmp); ++j) { const size_t folderIndex = j - folderCmp.begin(); const FolderPairSyncCfg& folderPairCfg = syncConfig[folderIndex]; delHandlerL.emplace_back(folderPairCfg.handleDeletion, folderPairCfg.versioningFolder, timeStamp, folderPairCfg.versionCountLimit_, folderIndex, j->getBaseDirPf(), callback); delHandlerR.emplace_back(folderPairCfg.handleDeletion, folderPairCfg.versioningFolder, timeStamp, folderPairCfg.versionCountLimit_, folderIndex, j->getBaseDirPf(), callback); } //-------------------execute basic checks all at once before starting sync-------------------------------------- auto dependentDir = [](const Zstring& lhs, const Zstring& rhs) //note: this is NOT an equivalence relation! { return EqualFilename()(Zstring(lhs.c_str(), std::min(lhs.length(), rhs.length())), Zstring(rhs.c_str(), std::min(lhs.length(), rhs.length()))); }; //aggregate information std::map> dirReadWriteCount; //count read/write accesses auto incReadCount = [&](const Zstring& baseDir) { dirReadWriteCount[baseDir]; //create entry for (auto iter = dirReadWriteCount.begin(); iter != dirReadWriteCount.end(); ++iter) { auto& countRef = iter->second; if (dependentDir(baseDir, iter->first)) ++countRef.first; } }; auto incWriteCount = [&](const Zstring& baseDir) { dirReadWriteCount[baseDir]; //create entry for (auto iter = dirReadWriteCount.begin(); iter != dirReadWriteCount.end(); ++iter) { auto& countRef = iter->second; if (dependentDir(baseDir, iter->first)) ++countRef.second; } }; typedef std::vector> DirPairList; DirPairList significantDiff; typedef std::vector>> DirSpaceRequAvailList; //dirname / space required / space available DirSpaceRequAvailList diskSpaceMissing; #ifdef FFS_WIN std::set recyclMissing; #endif //start checking folder pairs { auto iterDelHandlerL = delHandlerL.cbegin(); auto iterDelHandlerR = delHandlerR.cbegin(); for (auto j = begin(folderCmp); j != end(folderCmp); ++j, ++iterDelHandlerL, ++iterDelHandlerR) { const size_t folderIndex = j - begin(folderCmp); //exclude some pathological case (leftdir, rightdir are empty) if (EqualFilename()(j->getBaseDirPf(), j->getBaseDirPf())) continue; const FolderPairSyncCfg& folderPairCfg = syncConfig[folderIndex]; const SyncStatistics folderPairStat(*j); //aggregate basic information const bool writeLeft = folderPairStat.getCreate() + folderPairStat.getUpdate() + folderPairStat.getDelete() > 0; const bool writeRight = folderPairStat.getCreate() + folderPairStat.getUpdate() + folderPairStat.getDelete() > 0; //skip folder pair if there is nothing to do (except for automatic mode, where data base needs to be written even in this case) if (!writeLeft && !writeRight && !folderPairCfg.inAutomaticMode) { skipFolderPair[folderIndex] = true; //skip creating (not yet existing) base directories in particular if there's no need continue; } //check empty input fields: basically this only makes sense if empty field is not target (and not automatic mode: because of db file creation) if ((j->getBaseDirPf().empty() && (writeLeft || folderPairCfg.inAutomaticMode)) || (j->getBaseDirPf().empty() && (writeRight || folderPairCfg.inAutomaticMode))) { callback.reportFatalError(_("Target folder input field must not be empty.")); skipFolderPair[folderIndex] = true; continue; } //aggregate information of folders used by multiple pairs in read/write access if (!dependentDir(j->getBaseDirPf(), j->getBaseDirPf())) //true in general { if (writeLeft && writeRight) { incWriteCount(j->getBaseDirPf()); incWriteCount(j->getBaseDirPf()); } else if (writeLeft) { incWriteCount(j->getBaseDirPf()); incReadCount (j->getBaseDirPf()); } else if (writeRight) { incReadCount (j->getBaseDirPf()); incWriteCount(j->getBaseDirPf()); } } else //if folder pair contains two dependent folders, a warning was already issued after comparison; in this context treat as one write access at most { if (writeLeft || writeRight) incWriteCount(j->getBaseDirPf()); } if (folderPairStat.getUpdate() + folderPairStat.getDelete() > 0 && folderPairCfg.handleDeletion == zen::DELETE_TO_VERSIONING) { //check if user-defined directory for deletion was specified if (folderPairCfg.versioningFolder.empty()) //already trimmed by getFormattedDirectoryName() { callback.reportFatalError(_("Folder input field for versioning must not be empty.")); skipFolderPair[folderIndex] = true; continue; } } //the following scenario is covered by base directory creation below in case source directory exists (accessible or not), but latter doesn't cover not-yet-created source!!! auto checkSourceMissing = [&](const Zstring& baseDirPf, bool wasExisting) -> bool //avoid race-condition: we need to evaluate existence status from time of comparison! { const Zstring dirname = beforeLast(baseDirPf, FILE_NAME_SEPARATOR); if (!dirname.empty()) { //PERMANENT network drop: avoid data loss when source directory is not found AND user chose to ignore errors (else we wouldn't arrive here) if (folderPairStat.getCreate() + folderPairStat.getUpdate() == 0 && folderPairStat.getDelete() > 0) //deletions only... (respect filtered items!) //folderPairStat.getConflict() == 0 && -> there COULD be conflicts for if directory existence check fails, but loading sync.ffs_db succeeds //https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3531351&group_id=234430 -> fixed, but still better not consider conflicts { if (!wasExisting) //avoid race-condition: we need to evaluate existence status from time of comparison! { callback.reportFatalError(replaceCpy(_("Source folder %x not found."), L"%x", fmtFileName(baseDirPf))); skipFolderPair[folderIndex] = true; return false; } } } return true; }; if (!checkSourceMissing(j->getBaseDirPf(), j->isExisting()) || !checkSourceMissing(j->getBaseDirPf(), j->isExisting())) continue; //check if more than 50% of total number of files/dirs are to be created/overwritten/deleted if (significantDifferenceDetected(folderPairStat)) significantDiff.push_back(std::make_pair(j->getBaseDirPf(), j->getBaseDirPf())); //check for sufficient free diskspace auto checkSpace = [&](const Zstring& baseDirPf, const Int64& spaceRequired) { try { Int64 freeSpace = to(getFreeDiskSpace(baseDirPf)); //throw FileError if (0 < freeSpace && //zero disk space is either an error or not: in both cases this warning message is obsolete (WebDav seems to report 0) freeSpace < spaceRequired) diskSpaceMissing.push_back(std::make_pair(baseDirPf, std::make_pair(spaceRequired, freeSpace))); } catch (FileError&) {} }; const std::pair spaceNeeded = DiskSpaceNeeded::calculate(*j, iterDelHandlerL->deletionFreesSpace(), iterDelHandlerR->deletionFreesSpace()); checkSpace(j->getBaseDirPf(), spaceNeeded.first); checkSpace(j->getBaseDirPf(), spaceNeeded.second); #ifdef FFS_WIN //windows: check if recycle bin really exists; if not, Windows will silently delete, which is wrong if (folderPairCfg.handleDeletion == DELETE_TO_RECYCLER) { if (folderPairStat.getUpdate() + folderPairStat.getDelete() > 0 && iterDelHandlerL->recyclerFallbackOnDelete()) recyclMissing.insert(j->getBaseDirPf()); if (folderPairStat.getUpdate() + folderPairStat.getDelete() > 0 && iterDelHandlerR->recyclerFallbackOnDelete()) recyclMissing.insert(j->getBaseDirPf()); } #endif } } //check if unresolved conflicts exist if (statisticsTotal.getConflict() > 0) { //show the first few conflicts in warning message also: std::wstring warningMessage = _("Unresolved conflicts existing!") + L" (" + toGuiString(statisticsTotal.getConflict()) + L")\n\n"; const auto& conflictMsgs = statisticsTotal.getConflictMessages(); //get first few sync conflicts for (auto iter = conflictMsgs.begin(); iter != conflictMsgs.end(); ++iter) warningMessage += fmtFileName(iter->first) + L": " + iter->second + L"\n\n"; // if (statisticsTotal.getConflict() > static_cast(conflictMsgs.size())) // warningMessage += L"[...]\n\n"; warningMessage += _("You can ignore conflicts and continue synchronization."); callback.reportWarning(warningMessage, warnings.warningUnresolvedConflicts); } //check if user accidentally selected wrong directories for sync if (!significantDiff.empty()) { std::wstring warningMessage = _("Significant difference detected:"); for (auto iter = significantDiff.begin(); iter != significantDiff.end(); ++iter) warningMessage += std::wstring(L"\n\n") + iter->first + L" <-> " + L"\n" + iter->second; warningMessage += L"\n\n"; warningMessage += _("More than 50% of the total number of files will be copied or deleted!"); callback.reportWarning(warningMessage, warnings.warningSignificantDifference); } //check for sufficient free diskspace if (!diskSpaceMissing.empty()) { std::wstring warningMessage = _("Not enough free disk space available in:"); for (auto i = diskSpaceMissing.begin(); i != diskSpaceMissing.end(); ++i) warningMessage += std::wstring(L"\n\n") + fmtFileName(i->first) + L"\n" + _("Required:") + L" " + filesizeToShortString(i->second.first) + L"\n" + _("Available:") + L" " + filesizeToShortString(i->second.second); callback.reportWarning(warningMessage, warnings.warningNotEnoughDiskSpace); } //windows: check if recycle bin really exists; if not, Windows will silently delete, which is wrong #ifdef FFS_WIN if (!recyclMissing.empty()) { std::wstring warningMessage = _("Recycle Bin is not available for the following paths! Files will be deleted permanently instead:"); warningMessage += L"\n"; std::for_each(recyclMissing.begin(), recyclMissing.end(), [&](const Zstring& path) { warningMessage += L"\n" + utfCvrtTo(path); }); callback.reportWarning(warningMessage, warnings.warningRecyclerMissing); } #endif //check if folders are used by multiple pairs in read/write access std::vector conflictDirs; for (auto iter = dirReadWriteCount.cbegin(); iter != dirReadWriteCount.cend(); ++iter) { const std::pair& countRef = iter->second; //# read/write accesses if (countRef.first + countRef.second >= 2 && countRef.second >= 1) //race condition := multiple accesses of which at least one is a write conflictDirs.push_back(iter->first); } if (!conflictDirs.empty()) { std::wstring warningMessage = _("A folder will be modified which is part of multiple folder pairs. Please review synchronization settings.") + L"\n"; for (auto i = conflictDirs.begin(); i != conflictDirs.end(); ++i) warningMessage += L"\n" + fmtFileName(*i); callback.reportWarning(warningMessage, warnings.warningFolderPairRaceCondition); } //-------------------end of basic checks------------------------------------------ #ifdef FFS_WIN //shadow copy buffer: per sync-instance, not folder pair std::unique_ptr shadowCopyHandler(copyLockedFiles ? new shadow::ShadowCopy : nullptr); #endif try { //loop through all directory pairs auto iterDelHandlerL = delHandlerL.begin(); auto iterDelHandlerR = delHandlerR.begin(); for (auto j = begin(folderCmp); j != end(folderCmp); ++j, ++iterDelHandlerL, ++iterDelHandlerR) { //exclude some pathological case (leftdir, rightdir are empty) if (EqualFilename()(j->getBaseDirPf(), j->getBaseDirPf())) continue; //------------------------------------------------------------------------------------------ //always report folder pairs for log file, even if there is no work to do std::wstring left = _("Left") + L": "; std::wstring right = _("Right") + L": "; makeSameLength(left, right); callback.reportInfo(_("Synchronizing folder pair:") + L"\n" + L" " + left + fmtFileName(j->getBaseDirPf()) + L"\n" + L" " + right + fmtFileName(j->getBaseDirPf())); //------------------------------------------------------------------------------------------ const size_t folderIndex = j - begin(folderCmp); const FolderPairSyncCfg& folderPairCfg = syncConfig[folderIndex]; if (skipFolderPair[folderIndex]) //folder pairs may be skipped after fatal errors were found continue; //create base directories first (if not yet existing) -> no symlink or attribute copying! if (!createBaseDirectory(*j, callback) || !createBaseDirectory(*j, callback)) continue; //skip this folder pair //------------------------------------------------------------------------------------------ //execute synchronization recursively //update synchronization database (automatic sync only) ScopeGuard guardUpdateDb = makeGuard([&] { if (folderPairCfg.inAutomaticMode) try { zen::saveLastSynchronousState(*j); } //throw FileError catch (...) {} }); //guarantee removal of invalid entries (where element on both sides is empty) ZEN_ON_SCOPE_EXIT(BaseDirMapping::removeEmpty(*j);); bool copyPermissionsFp = false; tryReportingError([&] { copyPermissionsFp = copyFilePermissions && //copy permissions only if asked for and supported by *both* sides! !j->getBaseDirPf().empty() && //scenario: directory selected on one side only !j->getBaseDirPf().empty() && // supportsPermissions(beforeLast(j->getBaseDirPf(), FILE_NAME_SEPARATOR)) && //throw FileError supportsPermissions(beforeLast(j->getBaseDirPf(), FILE_NAME_SEPARATOR)); }, callback); //show error dialog if necessary SynchronizeFolderPair syncFP(callback, verifyCopiedFiles, copyPermissionsFp, transactionalFileCopy, #ifdef FFS_WIN shadowCopyHandler.get(), #endif *iterDelHandlerL, *iterDelHandlerR); syncFP.startSync(*j); //(try to gracefully) cleanup temporary Recycle bin folders and versioning -> will be done in ~DeletionHandling anyway... tryReportingError([&] { iterDelHandlerL->tryCleanup(); }, callback); //show error dialog if necessary tryReportingError([&] { iterDelHandlerR->tryCleanup(); }, callback); // //(try to gracefully) write database file (will be done in ~EnforceUpdateDatabase anyway...) if (folderPairCfg.inAutomaticMode) { callback.reportStatus(_("Generating database...")); callback.forceUiRefresh(); tryReportingError([&] { zen::saveLastSynchronousState(*j); }, callback); //throw FileError guardUpdateDb.dismiss(); } } } catch (const std::exception& e) { callback.reportFatalError(utfCvrtTo(e.what())); } } //########################################################################################### //callback functionality for smooth progress indicators template class WhileCopying : public zen::CallbackCopyFile { public: WhileCopying(Int64& bytesReported, ProcessCallback& statusHandler, const DelTargetCommand& cmd) : bytesReported_(bytesReported), statusHandler_(statusHandler), cmd_(cmd) {} virtual void deleteTargetFile(const Zstring& targetFile) { cmd_(); } virtual void updateCopyStatus(Int64 bytesDelta) { //inform about the (differential) processed amount of data statusHandler_.updateProcessedData(0, bytesDelta); //throw()! -> this ensures client and service provider are in sync! bytesReported_ += bytesDelta; // statusHandler_.requestUiRefresh(); //may throw } private: Int64& bytesReported_; ProcessCallback& statusHandler_; DelTargetCommand cmd_; }; //copy file while refreshing UI template void SynchronizeFolderPair::copyFileUpdatingTo(const FileMapping& fileObj, const DelTargetCommand& cmd, FileAttrib& newAttr) const { const Int64 expectedBytesToCpy = to(fileObj.getFileSize::result>()); Zstring source = fileObj.getFullName::result>(); const Zstring& target = fileObj.getBaseDirPf() + fileObj.getRelativeName::result>(); Int64 bytesReported; auto copyOperation = [&] { //start of (possibly) long-running copy process: ensure status updates are performed regularly //in error situation: undo communication of processed amount of data ScopeGuard guardStatistics = makeGuard([&] { procCallback_.updateProcessedData(0, -bytesReported); bytesReported = 0; }); WhileCopying callback(bytesReported, procCallback_, cmd); copyFile(source, //type File implicitly means symlinks need to be dereferenced! target, copyFilePermissions_, transactionalFileCopy_, &callback, &newAttr); //throw FileError, ErrorFileLocked //#################### Verification ############################# if (verifyCopiedFiles_) { ScopeGuard guardTarget = makeGuard([&] { removeFile(target); }); //delete target if verification fails verifyFileCopy(source, target); //throw FileError guardTarget.dismiss(); } //#################### /Verification ############################# //inform about the (remaining) processed amount of data if (bytesReported != expectedBytesToCpy) procCallback_.updateTotalData(0, bytesReported - expectedBytesToCpy); //we model physical statistic numbers => adjust total: consider ADS, sparse, compressed files -> transferred bytes may differ from file size (which is just a rough guess)! guardStatistics.dismiss(); }; #ifdef FFS_WIN try { copyOperation(); } catch (ErrorFileLocked& e1) { //if file is locked (try to) use Windows Volume Shadow Copy Service if (!shadowCopyHandler_) throw; try { //contains prefix: E.g. "\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1\Program Files\FFS\sample.dat" source = shadowCopyHandler_->makeShadowCopy(source); //throw FileError } catch (const FileError& e2) { throw FileError(e1.toString() + L"\n\n" + e2.toString()); } //now try again copyOperation(); } #else copyOperation(); #endif } //--------------------- data verification ------------------------- struct VerifyCallback { virtual ~VerifyCallback() {} virtual void updateStatus() = 0; }; void verifyFiles(const Zstring& source, const Zstring& target, VerifyCallback& callback) // throw (FileError) { static std::vector memory1(1024 * 1024); //1024 kb seems to be a reasonable buffer size static std::vector memory2(1024 * 1024); #ifdef FFS_WIN wxFile file1(applyLongPathPrefix(source).c_str(), wxFile::read); //don't use buffered file input for verification! #elif defined FFS_LINUX wxFile file1(::open(source.c_str(), O_RDONLY)); //utilize UTF-8 filename #endif if (!file1.IsOpened()) throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(source))); #ifdef FFS_WIN wxFile file2(applyLongPathPrefix(target).c_str(), wxFile::read); //don't use buffered file input for verification! #elif defined FFS_LINUX wxFile file2(::open(target.c_str(), O_RDONLY)); //utilize UTF-8 filename #endif if (!file2.IsOpened()) //NO cleanup necessary for (wxFile) file1 throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(target))); do { const size_t length1 = file1.Read(&memory1[0], memory1.size()); if (file1.Error()) throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(source)) + L" (r)"); callback.updateStatus(); //send progress updates const size_t length2 = file2.Read(&memory2[0], memory2.size()); if (file2.Error()) throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(target)) + L" (r)"); callback.updateStatus(); //send progress updates if (length1 != length2 || ::memcmp(&memory1[0], &memory2[0], length1) != 0) throw FileError(_("Data verification error: Source and target file have different content!") + L"\n" + fmtFileName(source) + L" -> \n" + fmtFileName(target)); } while (!file1.Eof()); if (!file2.Eof()) throw FileError(_("Data verification error: Source and target file have different content!") + L"\n" + fmtFileName(source) + L" -> \n" + fmtFileName(target)); } class VerifyStatusUpdater : public VerifyCallback { public: VerifyStatusUpdater(ProcessCallback& statusHandler) : statusHandler_(statusHandler) {} virtual void updateStatus() { statusHandler_.requestUiRefresh(); } //trigger display refresh private: ProcessCallback& statusHandler_; }; void SynchronizeFolderPair::verifyFileCopy(const Zstring& source, const Zstring& target) const { procCallback_.reportInfo(replaceCpy(txtVerifying, L"%x", fmtFileName(target))); VerifyStatusUpdater callback(procCallback_); tryReportingError([&] { ::verifyFiles(source, target, callback); }, procCallback_); }