diff options
| author | B. Stack <bgstack15@gmail.com> | 2022-10-11 11:16:39 -0400 |
|---|---|---|
| committer | B. Stack <bgstack15@gmail.com> | 2022-10-11 11:16:39 -0400 |
| commit | cab22f2dc3c5f41b5163f74cbb233e390edff6ff (patch) | |
| tree | a49cfd729d9793681a57fa6f7409b0f0848e9ede | |
| parent | Merge branch 'b11.25' into 'master' (diff) | |
| download | FreeFileSync-cab22f2dc3c5f41b5163f74cbb233e390edff6ff.tar.gz FreeFileSync-cab22f2dc3c5f41b5163f74cbb233e390edff6ff.tar.bz2 FreeFileSync-cab22f2dc3c5f41b5163f74cbb233e390edff6ff.zip | |
add upstream 11.26
93 files changed, 3278 insertions, 2792 deletions
diff --git a/Changelog.txt b/Changelog.txt index f0a3df5c..c3709227 100644 --- a/Changelog.txt +++ b/Changelog.txt @@ -1,3 +1,17 @@ +FreeFileSync 11.26 [2022-10-06] +------------------------------- +Faster file copy for SSD-based hard drives (Linux, macOS) +Don't fill the OS file cache during file copy (macOS) +Removed redundant memory buffering during file copy +Fixed ERROR_FILE_EXISTS on Samba share when copying files with NTFS extended attributes +Show warning when recycle bin is not available (macOS, Linux) +Customize config item background colors +Fixed macOS menu bar not showing after app start +Fixed normalizing strings with broken UTF encoding +Fixed sound playback not working (Linux) +Don't allow creating file names ending with dot character (Windows) + + FreeFileSync 11.25 [2022-08-31] ------------------------------- Fixed crash when normalizing Unicode non-characters diff --git a/FreeFileSync/Build/Resources/Icons.zip b/FreeFileSync/Build/Resources/Icons.zip Binary files differindex 7de873b1..3817e263 100644 --- a/FreeFileSync/Build/Resources/Icons.zip +++ b/FreeFileSync/Build/Resources/Icons.zip diff --git a/FreeFileSync/Build/Resources/Languages.zip b/FreeFileSync/Build/Resources/Languages.zip Binary files differindex ec222bbc..d1126adf 100644 --- a/FreeFileSync/Build/Resources/Languages.zip +++ b/FreeFileSync/Build/Resources/Languages.zip diff --git a/FreeFileSync/Source/RealTimeSync/application.cpp b/FreeFileSync/Source/RealTimeSync/application.cpp index 8928ab5d..087e04ba 100644 --- a/FreeFileSync/Source/RealTimeSync/application.cpp +++ b/FreeFileSync/Source/RealTimeSync/application.cpp @@ -13,6 +13,7 @@ #include <wx/event.h> #include <wx/log.h> #include <wx/tooltip.h> +#include <wx+/app_main.h> #include <wx+/popup_dlg.h> #include <wx+/image_resources.h> #include "config.h" @@ -59,7 +60,7 @@ void notifyAppError(const std::wstring& msg, FfsExitCode rc) (msgTypeName.empty() ? L"" : SPACED_DASH + msgTypeName); //error handling strategy unknown and no sync log output available at this point! - std::cerr << '[' + utfTo<std::string>(title) + "] " + utfTo<std::string>(msg) << '\n'; + std::cerr << '[' + utfTo<std::string>(title) + "] " + utfTo<std::string>(msg) + '\n'; //alternative0: std::wcerr: cannot display non-ASCII at all, so why does it exist??? //alternative1: wxSafeShowMessage => NO console output on Debian x86, WTF! //alternative2: wxMessageBox() => works, but we probably shouldn't block during command line usage @@ -169,25 +170,41 @@ void Application::onEnterEventLoop(wxEvent& event) [[maybe_unused]] bool ubOk = Unbind(EVENT_ENTER_EVENT_LOOP, &Application::onEnterEventLoop, this); assert(ubOk); + //wxWidgets app exit handling is weird... we want to exit only if the logical main window is closed, not just *any* window! + wxTheApp->SetExitOnFrameDelete(false); //prevent popup-windows from becoming temporary top windows leading to program exit after closure + ZEN_ON_SCOPE_EXIT(if (!globalWindowWasSet()) wxTheApp->ExitMainLoop()); //quit application, if no main window was set (batch silent mode) + //try to set config/batch- filepath set by %1 parameter std::vector<Zstring> commandArgs; - for (int i = 1; i < argc; ++i) + + try { - const Zstring& filePath = getResolvedFilePath(utfTo<Zstring>(argv[i])); + for (int i = 1; i < argc; ++i) + { + const Zstring& filePath = getResolvedFilePath(utfTo<Zstring>(argv[i])); #if 0 - if (!fileAvailable(filePath)) //...be a little tolerant - for (const Zchar* ext : {Zstr(".ffs_real"), Zstr(".ffs_batch")}) - if (fileAvailable(filePath + ext)) - filePath += ext; + if (!fileAvailable(filePath)) //...be a little tolerant + for (const Zchar* ext : {Zstr(".ffs_real"), Zstr(".ffs_batch")}) + if (fileAvailable(filePath + ext)) + filePath += ext; #endif - commandArgs.push_back(filePath); - } + if (endsWithAsciiNoCase(filePath, Zstr(".ffs_real"))) + commandArgs.push_back(filePath); + else + throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath)), + _("Unexpected file extension:") + L' ' + fmtPath(getFileExtension(filePath))); + } - Zstring cfgFilePath; - if (!commandArgs.empty()) - cfgFilePath = commandArgs[0]; + Zstring cfgFilePath; + if (!commandArgs.empty()) + cfgFilePath = commandArgs[0]; - MainDialog::create(cfgFilePath); + MainDialog::create(cfgFilePath); + } + catch (const FileError& e) + { + notifyAppError(e.toString(), FfsExitCode::aborted); + } } diff --git a/FreeFileSync/Source/RealTimeSync/config.cpp b/FreeFileSync/Source/RealTimeSync/config.cpp index ff5cf29f..95dd3cf5 100644 --- a/FreeFileSync/Source/RealTimeSync/config.cpp +++ b/FreeFileSync/Source/RealTimeSync/config.cpp @@ -93,6 +93,7 @@ void rts::readConfig(const Zstring& filePath, XmlRealConfig& cfg, std::wstring& if (formatVer < XML_FORMAT_RTS_CFG) try { rts::writeConfig(cfg, filePath); /*throw FileError*/ } catch (FileError&) { assert(false); } //don't bother user! + warn_static("at least log on failure!") } catch (const FileError& e) { warningMsg = e.toString(); } } diff --git a/FreeFileSync/Source/RealTimeSync/main_dlg.cpp b/FreeFileSync/Source/RealTimeSync/main_dlg.cpp index d240dd0f..95a4bc4b 100644 --- a/FreeFileSync/Source/RealTimeSync/main_dlg.cpp +++ b/FreeFileSync/Source/RealTimeSync/main_dlg.cpp @@ -7,6 +7,7 @@ #include "main_dlg.h" #include <wx/wupdlock.h> #include <wx/filedlg.h> +#include <wx+/app_main.h> #include <wx+/bitmap_button.h> #include <wx+/font_size.h> #include <wx+/popup_dlg.h> @@ -96,6 +97,9 @@ MainDialog::MainDialog(const Zstring& cfgFilePath) : Bind(wxEVT_CHAR_HOOK, [this](wxKeyEvent& event) { onLocalKeyEvent(event); }); + //notify about (logical) application main window => program won't quit, but stay on this dialog + setGlobalWindow(this); + //prepare drag & drop firstFolderPanel_ = std::make_unique<FolderSelector2>(this, *m_panelMainFolder, *m_buttonSelectFolderMain, *m_txtCtrlDirectoryMain, folderLastSelected_, m_staticTextFinalPath); diff --git a/FreeFileSync/Source/afs/abstract.cpp b/FreeFileSync/Source/afs/abstract.cpp index f7bd8bbb..368860b7 100644 --- a/FreeFileSync/Source/afs/abstract.cpp +++ b/FreeFileSync/Source/afs/abstract.cpp @@ -36,19 +36,19 @@ std::weak_ordering AFS::compareDevice(const AbstractFileSystem& lhs, const Abstr } -std::optional<AbstractPath> AFS::getParentPath(const AbstractPath& ap) +std::optional<AbstractPath> AFS::getParentPath(const AbstractPath& itemPath) { - if (const std::optional<AfsPath> parentAfsPath = getParentPath(ap.afsPath)) - return AbstractPath(ap.afsDevice, *parentAfsPath); + if (const std::optional<AfsPath> parentAfsPath = getParentPath(itemPath.afsPath)) + return AbstractPath(itemPath.afsDevice, *parentAfsPath); return {}; } -std::optional<AfsPath> AFS::getParentPath(const AfsPath& afsPath) +std::optional<AfsPath> AFS::getParentPath(const AfsPath& itemPath) { - if (!afsPath.value.empty()) - return AfsPath(beforeLast(afsPath.value, FILE_NAME_SEPARATOR, IfNotFoundReturn::none)); + if (!itemPath.value.empty()) + return AfsPath(beforeLast(itemPath.value, FILE_NAME_SEPARATOR, IfNotFoundReturn::none)); return {}; } @@ -80,109 +80,123 @@ private: } -void AFS::traverseFolderFlat(const AfsPath& afsPath, //throw FileError +void AFS::traverseFolderFlat(const AfsPath& folderPath, //throw FileError const std::function<void (const FileInfo& fi)>& onFile, const std::function<void (const FolderInfo& fi)>& onFolder, const std::function<void (const SymlinkInfo& si)>& onSymlink) const { auto ft = std::make_shared<FlatTraverserCallback>(onFile, onFolder, onSymlink); //throw FileError - traverseFolderRecursive({{afsPath, ft}}, 1 /*parallelOps*/); //throw FileError + traverseFolderRecursive({{folderPath, ft}}, 1 /*parallelOps*/); //throw FileError } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) -AFS::FileCopyResult AFS::copyFileAsStream(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, const IoCallback& notifyUnbufferedIO /*throw X*/) const +AFS::FileCopyResult AFS::copyFileAsStream(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, const IoCallback& notifyUnbufferedIO /*throw X*/) const { - int64_t totalUnbufferedIO = 0; - IOCallbackDivider cbd(notifyUnbufferedIO, totalUnbufferedIO); + int64_t totalBytesNotified = 0; + IOCallbackDivider notifyIoDiv(notifyUnbufferedIO, totalBytesNotified); int64_t totalBytesRead = 0; int64_t totalBytesWritten = 0; - auto notifyUnbufferedRead = [&](int64_t bytesDelta) { totalBytesRead += bytesDelta; cbd(bytesDelta); }; - auto notifyUnbufferedWrite = [&](int64_t bytesDelta) { totalBytesWritten += bytesDelta; cbd(bytesDelta); }; + IoCallback /*[!] not auto!*/ notifyUnbufferedRead = [&](int64_t bytesDelta) { totalBytesRead += bytesDelta; notifyIoDiv(bytesDelta); }; + IoCallback notifyUnbufferedWrite = [&](int64_t bytesDelta) { totalBytesWritten += bytesDelta; notifyIoDiv(bytesDelta); }; //-------------------------------------------------------------------------------------------------------- - auto streamIn = getInputStream(afsSource, notifyUnbufferedRead); //throw FileError, ErrorFileLocked + auto streamIn = getInputStream(sourcePath); //throw FileError, ErrorFileLocked StreamAttributes attrSourceNew = {}; //try to get the most current attributes if possible (input file might have changed after comparison!) - if (std::optional<StreamAttributes> attr = streamIn->getAttributesBuffered()) //throw FileError + if (std::optional<StreamAttributes> attr = streamIn->tryGetAttributesFast()) //throw FileError attrSourceNew = *attr; //Native/MTP/Google Drive else //use possibly stale ones: attrSourceNew = attrSource; //SFTP/FTP //TODO: evaluate: consequences of stale attributes //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - auto streamOut = getOutputStream(apTarget, attrSourceNew.fileSize, attrSourceNew.modTime, notifyUnbufferedWrite); //throw FileError + auto streamOut = getOutputStream(targetPath, attrSourceNew.fileSize, attrSourceNew.modTime); //throw FileError + + + unbufferedStreamCopy([&](void* buffer, size_t bytesToRead) + { + return streamIn->tryRead(buffer, bytesToRead, notifyUnbufferedRead); //throw FileError, ErrorFileLocked, X + }, + streamIn->getBlockSize() /*throw FileError*/, + + [&](const void* buffer, size_t bytesToWrite) + { + return streamOut->tryWrite(buffer, bytesToWrite, notifyUnbufferedWrite); //throw FileError, X + }, + streamOut->getBlockSize() /*throw FileError*/); //throw FileError, ErrorFileLocked, X - bufferedStreamCopy(*streamIn, *streamOut); //throw FileError, ErrorFileLocked, X //check incomplete input *before* failing with (slightly) misleading error message in OutputStream::finalize() if (totalBytesRead != makeSigned(attrSourceNew.fileSize)) - throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(afsSource))), + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(sourcePath))), replaceCpy(replaceCpy(_("Unexpected size of data stream.\nExpected: %x bytes\nActual: %y bytes"), L"%x", formatNumber(attrSourceNew.fileSize)), L"%y", formatNumber(totalBytesRead)) + L" [notifyUnbufferedRead]"); - const FinalizeResult finResult = streamOut->finalize(); //throw FileError, X + const FinalizeResult finResult = streamOut->finalize(notifyUnbufferedWrite); //throw FileError, X - ZEN_ON_SCOPE_FAIL(try { removeFilePlain(apTarget); /*throw FileError*/ } + ZEN_ON_SCOPE_FAIL(try { removeFilePlain(targetPath); /*throw FileError*/ } catch (FileError&) {}); //after finalize(): not guarded by ~AFS::OutputStream() anymore! + warn_static("log it!") //catch file I/O bugs + read/write conflicts: (note: different check than inside AFS::OutputStream::finalize() => checks notifyUnbufferedIO()!) if (totalBytesWritten != totalBytesRead) - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(apTarget))), + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(targetPath))), replaceCpy(replaceCpy(_("Unexpected size of data stream.\nExpected: %x bytes\nActual: %y bytes"), L"%x", formatNumber(totalBytesRead)), L"%y", formatNumber(totalBytesWritten)) + L" [notifyUnbufferedWrite]"); - FileCopyResult cpResult; - cpResult.fileSize = attrSourceNew.fileSize; - cpResult.modTime = attrSourceNew.modTime; - cpResult.sourceFilePrint = attrSourceNew.filePrint; - cpResult.targetFilePrint = finResult.filePrint; - cpResult.errorModTime = finResult.errorModTime; - /* Failing to set modification time is not a serious problem from synchronization perspective (treat like external update) - => Support additional scenarios: - - GVFS failing to set modTime for FTP: https://freefilesync.org/forum/viewtopic.php?t=2372 - - GVFS failing to set modTime for MTP: https://freefilesync.org/forum/viewtopic.php?t=2803 - - MTP failing to set modTime in general: fail non-silently rather than silently during file creation - - FTP failing to set modTime for servers without MFMT-support */ - return cpResult; + return + { + .fileSize = attrSourceNew.fileSize, + .modTime = attrSourceNew.modTime, + .sourceFilePrint = attrSourceNew.filePrint, + .targetFilePrint = finResult.filePrint, + .errorModTime = finResult.errorModTime, + /* Failing to set modification time is not a serious problem from synchronization perspective (treat like external update) + => Support additional scenarios: + - GVFS failing to set modTime for FTP: https://freefilesync.org/forum/viewtopic.php?t=2372 + - GVFS failing to set modTime for MTP: https://freefilesync.org/forum/viewtopic.php?t=2803 + - MTP failing to set modTime in general: fail non-silently rather than silently during file creation + - FTP failing to set modTime for servers without MFMT-support */ + }; } //already existing + no onDeleteTargetFile: undefined behavior! (e.g. fail/overwrite/auto-rename) -AFS::FileCopyResult AFS::copyFileTransactional(const AbstractPath& apSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, +AFS::FileCopyResult AFS::copyFileTransactional(const AbstractPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, bool copyFilePermissions, bool transactionalCopy, const std::function<void()>& onDeleteTargetFile, const IoCallback& notifyUnbufferedIO /*throw X*/) { - auto copyFilePlain = [&](const AbstractPath& apTargetTmp) + auto copyFilePlain = [&](const AbstractPath& targetPathTmp) { //caveat: typeid returns static type for pointers, dynamic type for references!!! - if (typeid(apSource.afsDevice.ref()) == typeid(apTargetTmp.afsDevice.ref())) - return apSource.afsDevice.ref().copyFileForSameAfsType(apSource.afsPath, attrSource, - apTargetTmp, copyFilePermissions, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X + if (typeid(sourcePath.afsDevice.ref()) == typeid(targetPathTmp.afsDevice.ref())) + return sourcePath.afsDevice.ref().copyFileForSameAfsType(sourcePath.afsPath, attrSource, + targetPathTmp, copyFilePermissions, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //fall back to stream-based file copy: if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(getDisplayPath(apTargetTmp))), + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(getDisplayPath(targetPathTmp))), _("Operation not supported between different devices.")); //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - return apSource.afsDevice.ref().copyFileAsStream(apSource.afsPath, attrSource, apTargetTmp, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X + return sourcePath.afsDevice.ref().copyFileAsStream(sourcePath.afsPath, attrSource, targetPathTmp, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X }; - if (transactionalCopy && !hasNativeTransactionalCopy(apTarget)) + if (transactionalCopy && !hasNativeTransactionalCopy(targetPath)) { - const std::optional<AbstractPath> parentPath = getParentPath(apTarget); + const std::optional<AbstractPath> parentPath = getParentPath(targetPath); if (!parentPath) - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(apTarget))), L"Path is device root."); - const Zstring fileName = getItemName(apTarget); + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(targetPath))), L"Path is device root."); + const Zstring fileName = getItemName(targetPath); //- generate (hopefully) unique file name to avoid clashing with some remnant ffs_tmp file //- do not loop: avoid pathological cases, e.g. https://freefilesync.org/forum/viewtopic.php?t=1592 @@ -194,21 +208,22 @@ AFS::FileCopyResult AFS::copyFileTransactional(const AbstractPath& apSource, con const Zstring& shortGuid = printNumber<Zstring>(Zstr("%04x"), static_cast<unsigned int>(getCrc16(generateGUID()))); - const AbstractPath apTargetTmp = appendRelPath(*parentPath, tmpName + Zstr('~') + shortGuid + TEMP_FILE_ENDING); + const AbstractPath targetPathTmp = appendRelPath(*parentPath, tmpName + Zstr('~') + shortGuid + TEMP_FILE_ENDING); //------------------------------------------------------------------------------------------- - const FileCopyResult result = copyFilePlain(apTargetTmp); //throw FileError, ErrorFileLocked + const FileCopyResult result = copyFilePlain(targetPathTmp); //throw FileError, ErrorFileLocked //transactional behavior: ensure cleanup; not needed before copyFilePlain() which is already transactional - ZEN_ON_SCOPE_FAIL( try { removeFilePlain(apTargetTmp); } + ZEN_ON_SCOPE_FAIL( try { removeFilePlain(targetPathTmp); } catch (FileError&) {}); + warn_static("log it!") //have target file deleted (after read access on source and target has been confirmed) => allow for almost transactional overwrite if (onDeleteTargetFile) onDeleteTargetFile(); //throw X //already existing: undefined behavior! (e.g. fail/overwrite) - moveAndRenameItem(apTargetTmp, apTarget); //throw FileError, (ErrorMoveUnsupported) + moveAndRenameItem(targetPathTmp, targetPath); //throw FileError, (ErrorMoveUnsupported) //perf: this call is REALLY expensive on unbuffered volumes! ~40% performance decrease on FAT USB stick! /* CAVEAT on FAT/FAT32: the sequence of deleting the target file and renaming "file.txt.ffs_tmp" to "file.txt" does @@ -228,20 +243,20 @@ AFS::FileCopyResult AFS::copyFileTransactional(const AbstractPath& apSource, con if (onDeleteTargetFile) onDeleteTargetFile(); - return copyFilePlain(apTarget); //throw FileError, ErrorFileLocked + return copyFilePlain(targetPath); //throw FileError, ErrorFileLocked } } -bool AFS::createFolderIfMissingRecursion(const AbstractPath& ap) //throw FileError +bool AFS::createFolderIfMissingRecursion(const AbstractPath& folderPath) //throw FileError { - const std::optional<AbstractPath> parentPath = getParentPath(ap); + const std::optional<AbstractPath> parentPath = getParentPath(folderPath); if (!parentPath) //device root return false; try //generally we expect that path already exists (see: versioning, base folder, log file path) => check first { - if (getItemType(ap) != ItemType::file) //throw FileError + if (getItemType(folderPath) != ItemType::file) //throw FileError return false; } catch (FileError&) {} //not yet existing or access error? let's find out... @@ -251,14 +266,14 @@ bool AFS::createFolderIfMissingRecursion(const AbstractPath& ap) //throw FileErr try { //already existing: fail - createFolderPlain(ap); //throw FileError + createFolderPlain(folderPath); //throw FileError return true; } catch (FileError&) { try { - if (getItemType(ap) != ItemType::file) //throw FileError + if (getItemType(folderPath) != ItemType::file) //throw FileError return true; //already existing => possible, if createFolderIfMissingRecursion() is run in parallel } catch (FileError&) {} //not yet existing or access error @@ -269,23 +284,23 @@ bool AFS::createFolderIfMissingRecursion(const AbstractPath& ap) //throw FileErr //default implementation: folder traversal -std::optional<AFS::ItemType> AFS::itemStillExists(const AfsPath& afsPath) const //throw FileError +std::optional<AFS::ItemType> AFS::itemStillExists(const AfsPath& itemPath) const //throw FileError { try { //fast check: 1. perf 2. expected by getFolderStatusNonBlocking() 3. traversing non-existing folder below MIGHT NOT FAIL (e.g. for SFTP on AWS) - return getItemType(afsPath); //throw FileError + return getItemType(itemPath); //throw FileError } catch (const FileError& e) //not existing or access error { - const std::optional<AfsPath> parentAfsPath = getParentPath(afsPath); + const std::optional<AfsPath> parentAfsPath = getParentPath(itemPath); if (!parentAfsPath) //device root throw; //else: let's dig deeper... don't bother checking Win32 codes; e.g. not existing item may have the codes: // ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND, ERROR_INVALID_NAME, ERROR_INVALID_DRIVE, // ERROR_NOT_READY, ERROR_INVALID_PARAMETER, ERROR_BAD_PATHNAME, ERROR_BAD_NETPATH => not reliable - const Zstring itemName = getItemName(afsPath); + const Zstring itemName = getItemName(itemPath); assert(!itemName.empty()); const std::optional<ItemType> parentType = itemStillExists(*parentAfsPath); //throw FileError @@ -308,26 +323,26 @@ std::optional<AFS::ItemType> AFS::itemStillExists(const AfsPath& afsPath) const //default implementation: folder traversal -void AFS::removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError +void AFS::removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const //one call for each object! { //deferred recursion => save stack space and allow deletion of extremely deep hierarchies! - std::function<void(const AfsPath& folderPath)> removeFolderRecursionImpl; - removeFolderRecursionImpl = [this, &onBeforeFileDeletion, &onBeforeFolderDeletion, &removeFolderRecursionImpl](const AfsPath& folderPath) //throw FileError + std::function<void(const AfsPath& folderPath2)> removeFolderRecursionImpl; + removeFolderRecursionImpl = [this, &onBeforeFileDeletion, &onBeforeFolderDeletion, &removeFolderRecursionImpl](const AfsPath& folderPath2) //throw FileError { std::vector<Zstring> fileNames; std::vector<Zstring> folderNames; std::vector<Zstring> symlinkNames; - traverseFolderFlat(folderPath, //throw FileError + traverseFolderFlat(folderPath2, //throw FileError [&](const FileInfo& fi) { fileNames.push_back(fi.itemName); }, [&](const FolderInfo& fi) { folderNames.push_back(fi.itemName); }, [&](const SymlinkInfo& si) { symlinkNames.push_back(si.itemName); }); for (const Zstring& fileName : fileNames) { - const AfsPath filePath(appendPath(folderPath.value, fileName)); + const AfsPath filePath(appendPath(folderPath2.value, fileName)); if (onBeforeFileDeletion) onBeforeFileDeletion(getDisplayPath(filePath)); //throw X @@ -336,7 +351,7 @@ void AFS::removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileErro for (const Zstring& symlinkName : symlinkNames) { - const AfsPath linkPath(appendPath(folderPath.value, symlinkName)); + const AfsPath linkPath(appendPath(folderPath2.value, symlinkName)); if (onBeforeFileDeletion) onBeforeFileDeletion(getDisplayPath(linkPath)); //throw X @@ -344,44 +359,86 @@ void AFS::removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileErro } for (const Zstring& folderName : folderNames) - removeFolderRecursionImpl(AfsPath(appendPath(folderPath.value, folderName))); //throw FileError + removeFolderRecursionImpl(AfsPath(appendPath(folderPath2.value, folderName))); //throw FileError if (onBeforeFolderDeletion) - onBeforeFolderDeletion(getDisplayPath(folderPath)); //throw X + onBeforeFolderDeletion(getDisplayPath(folderPath2)); //throw X - removeFolderPlain(folderPath); //throw FileError + removeFolderPlain(folderPath2); //throw FileError }; //-------------------------------------------------------------------------------------------------------------- //no error situation if directory is not existing! manual deletion relies on it! - if (std::optional<ItemType> type = itemStillExists(afsPath)) //throw FileError + if (std::optional<ItemType> type = itemStillExists(folderPath)) //throw FileError { if (*type == ItemType::symlink) { if (onBeforeFileDeletion) - onBeforeFileDeletion(getDisplayPath(afsPath)); //throw X + onBeforeFileDeletion(getDisplayPath(folderPath)); //throw X - removeSymlinkPlain(afsPath); //throw FileError + removeSymlinkPlain(folderPath); //throw FileError } else - removeFolderRecursionImpl(afsPath); //throw FileError + removeFolderRecursionImpl(folderPath); //throw FileError } else //even if the folder did not exist anymore, significant I/O work was done => report - if (onBeforeFolderDeletion) onBeforeFolderDeletion(getDisplayPath(afsPath)); //throw X + if (onBeforeFolderDeletion) onBeforeFolderDeletion(getDisplayPath(folderPath)); //throw X } -void AFS::removeFileIfExists(const AbstractPath& ap) //throw FileError +void AFS::removeFileIfExists(const AbstractPath& filePath) //throw FileError { try { - removeFilePlain(ap); //throw FileError + removeFilePlain(filePath); //throw FileError + } + catch (const FileError& e) + { + try + { + if (!itemStillExists(filePath)) //throw FileError + return; + } + //abstract context => unclear which exception is more relevant/useless: + catch (const FileError& e2) { throw FileError(replaceCpy(e.toString(), L"\n\n", L'\n'), replaceCpy(e2.toString(), L"\n\n", L'\n')); } + + throw; + } +} + + +void AFS::removeSymlinkIfExists(const AbstractPath& linkPath) //throw FileError +{ + try + { + removeSymlinkPlain(linkPath); //throw FileError + } + catch (const FileError& e) + { + try + { + if (!itemStillExists(linkPath)) //throw FileError + return; + } + //abstract context => unclear which exception is more relevant/useless: + catch (const FileError& e2) { throw FileError(replaceCpy(e.toString(), L"\n\n", L'\n'), replaceCpy(e2.toString(), L"\n\n", L'\n')); } + + throw; + } +} + + +void AFS::removeEmptyFolderIfExists(const AbstractPath& folderPath) //throw FileError +{ + try + { + removeFolderPlain(folderPath); //throw FileError } catch (const FileError& e) { try { - if (!itemStillExists(ap)) //throw FileError + if (!itemStillExists(folderPath)) //throw FileError return; } //abstract context => unclear which exception is more relevant/useless: @@ -392,17 +449,18 @@ void AFS::removeFileIfExists(const AbstractPath& ap) //throw FileError } -void AFS::removeSymlinkIfExists(const AbstractPath& ap) //throw FileError +void AFS::RecycleSession::moveToRecycleBinIfExists(const AbstractPath& itemPath, const Zstring& logicalRelPath) //throw FileError, RecycleBinUnavailable { try { - removeSymlinkPlain(ap); //throw FileError + moveToRecycleBin(itemPath, logicalRelPath); //throw FileError, RecycleBinUnavailable } + catch (RecycleBinUnavailable&) { throw; } //[!] no need for itemStillExists() file access! catch (const FileError& e) { try { - if (!itemStillExists(ap)) //throw FileError + if (!itemStillExists(itemPath)) //throw FileError return; } //abstract context => unclear which exception is more relevant/useless: @@ -413,17 +471,18 @@ void AFS::removeSymlinkIfExists(const AbstractPath& ap) //throw FileError } -void AFS::removeEmptyFolderIfExists(const AbstractPath& ap) //throw FileError +void AFS::moveToRecycleBinIfExists(const AbstractPath& itemPath) //throw FileError, RecycleBinUnavailable { try { - removeFolderPlain(ap); //throw FileError + moveToRecycleBin(itemPath); //throw FileError, RecycleBinUnavailable } + catch (RecycleBinUnavailable&) { throw; } //[!] no need for itemStillExists() file access! catch (const FileError& e) { try { - if (!itemStillExists(ap)) //throw FileError + if (!itemStillExists(itemPath)) //throw FileError return; } //abstract context => unclear which exception is more relevant/useless: diff --git a/FreeFileSync/Source/afs/abstract.h b/FreeFileSync/Source/afs/abstract.h index 0aae8bc0..7ce749f3 100644 --- a/FreeFileSync/Source/afs/abstract.h +++ b/FreeFileSync/Source/afs/abstract.h @@ -36,10 +36,10 @@ struct AfsPath //= path relative to the file system root folder (no leading/tral struct AbstractPath //THREAD-SAFETY: like an int! { - AbstractPath(const AfsDevice& afsIn, const AfsPath& afsPathIn) : afsDevice(afsIn), afsPath(afsPathIn) {} + AbstractPath(const AfsDevice& deviceIn, const AfsPath& pathIn) : afsDevice(deviceIn), afsPath(pathIn) {} //template <class T1, class T2> -> don't use forwarding constructor: it circumvents AfsPath's explicit constructor! - //AbstractPath(T1&& afsIn, T2&& afsPathIn) : afsDevice(std::forward<T1>(afsIn)), afsPath(std::forward<T2>(afsPathIn)) {} + //AbstractPath(T1&& deviceIn, T2&& pathIn) : afsDevice(std::forward<T1>(deviceIn)), afsPath(std::forward<T2>(pathIn)) {} AfsDevice afsDevice; //"const AbstractFileSystem" => all accesses expected to be thread-safe!!! AfsPath afsPath; //relative to device root @@ -49,36 +49,36 @@ struct AbstractPath //THREAD-SAFETY: like an int! struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model thread-safe access! { //=============== convenience ================= - static Zstring getItemName(const AbstractPath& ap) { assert(getParentPath(ap)); return getItemName(ap.afsPath); } - static Zstring getItemName(const AfsPath& afsPath) { using namespace zen; return afterLast(afsPath.value, FILE_NAME_SEPARATOR, IfNotFoundReturn::all); } + static Zstring getItemName(const AbstractPath& itemPath) { assert(getParentPath(itemPath)); return getItemName(itemPath.afsPath); } + static Zstring getItemName(const AfsPath& itemPath) { using namespace zen; return afterLast(itemPath.value, FILE_NAME_SEPARATOR, IfNotFoundReturn::all); } - static bool isNullPath(const AbstractPath& ap) { return isNullDevice(ap.afsDevice) /*&& ap.afsPath.value.empty()*/; } + static bool isNullPath(const AbstractPath& itemPath) { return isNullDevice(itemPath.afsDevice) /*&& itemPath.afsPath.value.empty()*/; } - static AbstractPath appendRelPath(const AbstractPath& ap, const Zstring& relPath); + static AbstractPath appendRelPath(const AbstractPath& itemPath, const Zstring& relPath); - static std::optional<AbstractPath> getParentPath(const AbstractPath& ap); - static std::optional<AfsPath> getParentPath(const AfsPath& afsPath); + static std::optional<AbstractPath> getParentPath(const AbstractPath& itemPath); + static std::optional<AfsPath> getParentPath(const AfsPath& itemPath); //============================================= static std::weak_ordering compareDevice(const AbstractFileSystem& lhs, const AbstractFileSystem& rhs); static bool isNullDevice(const AfsDevice& afsDevice) { return afsDevice.ref().isNullFileSystem(); } - static std::wstring getDisplayPath(const AbstractPath& ap) { return ap.afsDevice.ref().getDisplayPath(ap.afsPath); } + static std::wstring getDisplayPath(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().getDisplayPath(itemPath.afsPath); } - static Zstring getInitPathPhrase(const AbstractPath& ap) { return ap.afsDevice.ref().getInitPathPhrase(ap.afsPath); } + static Zstring getInitPathPhrase(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().getInitPathPhrase(itemPath.afsPath); } - static std::vector<Zstring> getPathPhraseAliases(const AbstractPath& ap) { return ap.afsDevice.ref().getPathPhraseAliases(ap.afsPath); } + static std::vector<Zstring> getPathPhraseAliases(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().getPathPhraseAliases(itemPath.afsPath); } //---------------------------------------------------------------------------------------------------------------- static void authenticateAccess(const AfsDevice& afsDevice, bool allowUserInteraction) //throw FileError { return afsDevice.ref().authenticateAccess(allowUserInteraction); } - static int getAccessTimeout(const AbstractPath& ap) { return ap.afsDevice.ref().getAccessTimeout(); } //returns "0" if no timeout in force + static int getAccessTimeout(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().getAccessTimeout(); } //returns "0" if no timeout in force - static bool supportPermissionCopy(const AbstractPath& apSource, const AbstractPath& apTarget); //throw FileError + static bool supportPermissionCopy(const AbstractPath& sourcePath, const AbstractPath& targetPath); //throw FileError - static bool hasNativeTransactionalCopy(const AbstractPath& ap) { return ap.afsDevice.ref().hasNativeTransactionalCopy(); } + static bool hasNativeTransactionalCopy(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().hasNativeTransactionalCopy(); } //---------------------------------------------------------------------------------------------------------------- using FingerPrint = uint64_t; //AfsDevice-dependent persistent unique ID @@ -91,43 +91,43 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t }; //(hopefully) fast: does not distinguish between error/not existing //root path? => do access test - static ItemType getItemType(const AbstractPath& ap) { return ap.afsDevice.ref().getItemType(ap.afsPath); } //throw FileError + static ItemType getItemType(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().getItemType(itemPath.afsPath); } //throw FileError //assumes: - base path still exists // - all child item path parts must correspond to folder traversal // => we can conclude whether an item is *not* existing anymore by doing a *case-sensitive* name search => potentially SLOW! // root path? => do access test - static std::optional<ItemType> itemStillExists(const AbstractPath& ap) { return ap.afsDevice.ref().itemStillExists(ap.afsPath); } //throw FileError + static std::optional<ItemType> itemStillExists(const AbstractPath& itemPath) { return itemPath.afsDevice.ref().itemStillExists(itemPath.afsPath); } //throw FileError //---------------------------------------------------------------------------------------------------------------- //already existing: fail //does NOT create parent directories recursively if not existing - static void createFolderPlain(const AbstractPath& ap) { ap.afsDevice.ref().createFolderPlain(ap.afsPath); } //throw FileError + static void createFolderPlain(const AbstractPath& folderPath) { folderPath.afsDevice.ref().createFolderPlain(folderPath.afsPath); } //throw FileError //creates directories recursively if not existing //returns false if folder already exists - static bool createFolderIfMissingRecursion(const AbstractPath& ap); //throw FileError + static bool createFolderIfMissingRecursion(const AbstractPath& folderPath); //throw FileError - static void removeFolderIfExistsRecursion(const AbstractPath& ap, //throw FileError + static void removeFolderIfExistsRecursion(const AbstractPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) //one call for each object! - { return ap.afsDevice.ref().removeFolderIfExistsRecursion(ap.afsPath, onBeforeFileDeletion, onBeforeFolderDeletion); } + { return folderPath.afsDevice.ref().removeFolderIfExistsRecursion(folderPath.afsPath, onBeforeFileDeletion, onBeforeFolderDeletion); } - static void removeFileIfExists (const AbstractPath& ap); // - static void removeSymlinkIfExists (const AbstractPath& ap); //throw FileError - static void removeEmptyFolderIfExists(const AbstractPath& ap); // + static void removeFileIfExists (const AbstractPath& filePath); // + static void removeSymlinkIfExists (const AbstractPath& linkPath); //throw FileError + static void removeEmptyFolderIfExists(const AbstractPath& folderPath); // - static void removeFilePlain (const AbstractPath& ap) { ap.afsDevice.ref().removeFilePlain (ap.afsPath); } // - static void removeSymlinkPlain(const AbstractPath& ap) { ap.afsDevice.ref().removeSymlinkPlain(ap.afsPath); } //throw FileError - static void removeFolderPlain (const AbstractPath& ap) { ap.afsDevice.ref().removeFolderPlain (ap.afsPath); } // + static void removeFilePlain (const AbstractPath& filePath ) { filePath .afsDevice.ref().removeFilePlain (filePath .afsPath); } // + static void removeSymlinkPlain(const AbstractPath& linkPath ) { linkPath .afsDevice.ref().removeSymlinkPlain(linkPath .afsPath); } //throw FileError + static void removeFolderPlain (const AbstractPath& folderPath) { folderPath.afsDevice.ref().removeFolderPlain (folderPath.afsPath); } // //---------------------------------------------------------------------------------------------------------------- - //static void setModTime(const AbstractPath& ap, time_t modTime) { ap.afsDevice.ref().setModTime(ap.afsPath, modTime); } //throw FileError, follows symlinks + //static void setModTime(const AbstractPath& itemPath, time_t modTime) { itemPath.afsDevice.ref().setModTime(itemPath.afsPath, modTime); } //throw FileError, follows symlinks - static AbstractPath getSymlinkResolvedPath(const AbstractPath& ap) { return ap.afsDevice.ref().getSymlinkResolvedPath (ap.afsPath); } //throw FileError - static bool equalSymlinkContent(const AbstractPath& apLhs, const AbstractPath& apRhs); //throw FileError + static AbstractPath getSymlinkResolvedPath(const AbstractPath& linkPath) { return linkPath.afsDevice.ref().getSymlinkResolvedPath(linkPath.afsPath); } //throw FileError + static bool equalSymlinkContent(const AbstractPath& linkPathL, const AbstractPath& linkPathR); //throw FileError //---------------------------------------------------------------------------------------------------------------- - static zen::FileIconHolder getFileIcon (const AbstractPath& ap, int pixelSize) { return ap.afsDevice.ref().getFileIcon (ap.afsPath, pixelSize); } //throw FileError; optional return value - static zen::ImageHolder getThumbnailImage(const AbstractPath& ap, int pixelSize) { return ap.afsDevice.ref().getThumbnailImage(ap.afsPath, pixelSize); } //throw FileError; optional return value + static zen::FileIconHolder getFileIcon (const AbstractPath& filePath, int pixelSize) { return filePath.afsDevice.ref().getFileIcon (filePath.afsPath, pixelSize); } //throw FileError; optional return value + static zen::ImageHolder getThumbnailImage(const AbstractPath& filePath, int pixelSize) { return filePath.afsDevice.ref().getThumbnailImage(filePath.afsPath, pixelSize); } //throw FileError; optional return value //---------------------------------------------------------------------------------------------------------------- struct StreamAttributes @@ -141,16 +141,17 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t struct InputStream { virtual ~InputStream() {} - virtual size_t read(void* buffer, size_t bytesToRead) = 0; //throw FileError, ErrorFileLocked, X; return "bytesToRead" bytes unless end of stream! - virtual size_t getBlockSize() const = 0; //non-zero block size is AFS contract! it's implementer's job to always give a reasonable buffer size! + virtual size_t getBlockSize() = 0; //throw FileError; non-zero block size is AFS contract! + virtual size_t tryRead(void* buffer, size_t bytesToRead, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) = 0; //throw FileError, ErrorFileLocked, X + //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! //only returns attributes if they are already buffered within stream handle and determination would be otherwise expensive (e.g. FTP/SFTP): - virtual std::optional<StreamAttributes> getAttributesBuffered() = 0; //throw FileError + virtual std::optional<StreamAttributes> tryGetAttributesFast() = 0; //throw FileError }; //return value always bound: - static std::unique_ptr<InputStream> getInputStream(const AbstractPath& ap, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, ErrorFileLocked - { return ap.afsDevice.ref().getInputStream(ap.afsPath, notifyUnbufferedIO); } + static std::unique_ptr<InputStream> getInputStream(const AbstractPath& filePath) { return filePath.afsDevice.ref().getInputStream(filePath.afsPath); } //throw FileError, ErrorFileLocked + //---------------------------------------------------------------------------------------------------------------- struct FinalizeResult { @@ -161,16 +162,18 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t struct OutputStreamImpl { virtual ~OutputStreamImpl() {} - virtual void write(const void* buffer, size_t bytesToWrite) = 0; //throw FileError, X - virtual FinalizeResult finalize() = 0; //throw FileError, X + virtual size_t getBlockSize() = 0; //throw FileError; non-zero block size is AFS contract + virtual size_t tryWrite(const void* buffer, size_t bytesToWrite, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) = 0; //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 + virtual FinalizeResult finalize(const zen::IoCallback& notifyUnbufferedIO /*throw X*/) = 0; //throw FileError, X }; struct OutputStream //call finalize when done! { OutputStream(std::unique_ptr<OutputStreamImpl>&& outStream, const AbstractPath& filePath, std::optional<uint64_t> streamSize); ~OutputStream(); - void write(const void* buffer, size_t bytesToWrite); //throw FileError, X - FinalizeResult finalize(); //throw FileError, X + size_t getBlockSize() { return outStream_->getBlockSize(); } //throw FileError + size_t tryWrite(const void* buffer, size_t bytesToWrite, const zen::IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, X may return short! + FinalizeResult finalize(const zen::IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, X private: std::unique_ptr<OutputStreamImpl> outStream_; //bound! @@ -180,11 +183,10 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t uint64_t bytesWrittenTotal_ = 0; }; //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - static std::unique_ptr<OutputStream> getOutputStream(const AbstractPath& ap, //throw FileError + static std::unique_ptr<OutputStream> getOutputStream(const AbstractPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const zen::IoCallback& notifyUnbufferedIO /*throw X*/) - { return std::make_unique<OutputStream>(ap.afsDevice.ref().getOutputStream(ap.afsPath, streamSize, modTime, notifyUnbufferedIO), ap, streamSize); } + std::optional<time_t> modTime) + { return std::make_unique<OutputStream>(filePath.afsDevice.ref().getOutputStream(filePath.afsPath, streamSize, modTime), filePath, streamSize); } //---------------------------------------------------------------------------------------------------------------- struct SymlinkInfo @@ -245,11 +247,11 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t //- client needs to handle duplicate file reports! (FilePlusTraverser fallback, retrying to read directory contents, ...) static void traverseFolderRecursive(const AfsDevice& afsDevice, const TraverserWorkload& workload /*throw X*/, size_t parallelOps) { afsDevice.ref().traverseFolderRecursive(workload, parallelOps); } - static void traverseFolderFlat(const AbstractPath& ap, //throw FileError + static void traverseFolderFlat(const AbstractPath& folderPath, //throw FileError const std::function<void (const FileInfo& fi)>& onFile, // const std::function<void (const FolderInfo& fi)>& onFolder, //optional const std::function<void (const SymlinkInfo& si)>& onSymlink) // - { ap.afsDevice.ref().traverseFolderFlat(ap.afsPath, onFile, onFolder, onSymlink); } + { folderPath.afsDevice.ref().traverseFolderFlat(folderPath.afsPath, onFile, onFolder, onSymlink); } //---------------------------------------------------------------------------------------------------------------- //already existing: undefined behavior! (e.g. fail/overwrite) @@ -272,8 +274,8 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t //symlink handling: follow //already existing + no onDeleteTargetFile: undefined behavior! (e.g. fail/overwrite/auto-rename) //returns current attributes at the time of copy - static FileCopyResult copyFileTransactional(const AbstractPath& apSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, + static FileCopyResult copyFileTransactional(const AbstractPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, bool copyFilePermissions, bool transactionalCopy, //if target is existing user *must* implement deletion to avoid undefined behavior @@ -284,30 +286,37 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t //already existing: fail //symlink handling: follow - static void copyNewFolder(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions); //throw FileError + static void copyNewFolder(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions); //throw FileError //already existing: fail - static void copySymlink(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions); //throw FileError + static void copySymlink(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions); //throw FileError //---------------------------------------------------------------------------------------------------------------- - static int64_t getFreeDiskSpace(const AbstractPath& ap) { return ap.afsDevice.ref().getFreeDiskSpace(ap.afsPath); } //throw FileError, returns < 0 if not available - - static bool supportsRecycleBin(const AbstractPath& ap) { return ap.afsDevice.ref().supportsRecycleBin(ap.afsPath); } //throw FileError + static int64_t getFreeDiskSpace(const AbstractPath& folderPath) { return folderPath.afsDevice.ref().getFreeDiskSpace(folderPath.afsPath); } //throw FileError, returns < 0 if not available struct RecycleSession { virtual ~RecycleSession() {} + //- multi-threaded access: internally synchronized! - virtual void recycleItemIfExists(const AbstractPath& itemPath, const Zstring& logicalRelPath) = 0; //throw FileError + void moveToRecycleBinIfExists(const AbstractPath& itemPath, const Zstring& logicalRelPath); //throw FileError, RecycleBinUnavailable + + //- fails if item is not existing + //- multi-threaded access: internally synchronized! + virtual void moveToRecycleBin(const AbstractPath& itemPath, const Zstring& logicalRelPath) = 0; //throw FileError, RecycleBinUnavailable virtual void tryCleanup(const std::function<void (const std::wstring& displayPath)>& notifyDeletionStatus /*throw X*; displayPath may be empty*/) = 0; //throw FileError, X }; - //precondition: supportsRecycleBin() must return true! - static std::unique_ptr<RecycleSession> createRecyclerSession(const AbstractPath& ap) { return ap.afsDevice.ref().createRecyclerSession(ap.afsPath); } //throw FileError, return value must be bound! + //return value always bound! + static std::unique_ptr<RecycleSession> createRecyclerSession(const AbstractPath& folderPath) { return folderPath.afsDevice.ref().createRecyclerSession(folderPath.afsPath); } //throw FileError, RecycleBinUnavailable + + //- returns empty on success, item type if recycle bin is not available + static void moveToRecycleBinIfExists(const AbstractPath& itemPath); //throw FileError, RecycleBinUnavailable - static void recycleItemIfExists(const AbstractPath& ap) { ap.afsDevice.ref().recycleItemIfExists(ap.afsPath); } //throw FileError + //fails if item is not existing + static void moveToRecycleBin(const AbstractPath& itemPath) { itemPath.afsDevice.ref().moveToRecycleBin(itemPath.afsPath); }; //throw FileError, RecycleBinUnavailable //================================================================================================================ @@ -317,87 +326,86 @@ struct AbstractFileSystem //THREAD-SAFETY: "const" member functions must model t protected: //default implementation: folder traversal - virtual std::optional<ItemType> itemStillExists(const AfsPath& afsPath) const = 0; //throw FileError + virtual std::optional<ItemType> itemStillExists(const AfsPath& itemPath) const = 0; //throw FileError //default implementation: folder traversal - virtual void removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError + virtual void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const = 0; //one call for each object! - void traverseFolderFlat(const AfsPath& afsPath, //throw FileError + void traverseFolderFlat(const AfsPath& folderPath, //throw FileError const std::function<void (const FileInfo& fi)>& onFile, // const std::function<void (const FolderInfo& fi)>& onFolder, //optional const std::function<void (const SymlinkInfo& si)>& onSymlink) const; // //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - FileCopyResult copyFileAsStream(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) const; + FileCopyResult copyFileAsStream(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) const; private: - virtual std::optional<Zstring> getNativeItemPath(const AfsPath& afsPath) const { return {}; }; + virtual std::optional<Zstring> getNativeItemPath(const AfsPath& itemPath) const { return {}; }; - virtual Zstring getInitPathPhrase(const AfsPath& afsPath) const = 0; + virtual Zstring getInitPathPhrase(const AfsPath& itemPath) const = 0; - virtual std::vector<Zstring> getPathPhraseAliases(const AfsPath& afsPath) const = 0; + virtual std::vector<Zstring> getPathPhraseAliases(const AfsPath& itemPath) const = 0; - virtual std::wstring getDisplayPath(const AfsPath& afsPath) const = 0; + virtual std::wstring getDisplayPath(const AfsPath& itemPath) const = 0; virtual bool isNullFileSystem() const = 0; virtual std::weak_ordering compareDeviceSameAfsType(const AbstractFileSystem& afsRhs) const = 0; //---------------------------------------------------------------------------------------------------------------- - virtual ItemType getItemType(const AfsPath& afsPath) const = 0; //throw FileError + virtual ItemType getItemType(const AfsPath& itemPath) const = 0; //throw FileError //---------------------------------------------------------------------------------------------------------------- //already existing: fail - virtual void createFolderPlain(const AfsPath& afsPath) const = 0; //throw FileError + virtual void createFolderPlain(const AfsPath& folderPath) const = 0; //throw FileError //non-recursive folder deletion: - virtual void removeFilePlain (const AfsPath& afsPath) const = 0; //throw FileError - virtual void removeSymlinkPlain(const AfsPath& afsPath) const = 0; //throw FileError - virtual void removeFolderPlain (const AfsPath& afsPath) const = 0; //throw FileError + virtual void removeFilePlain (const AfsPath& filePath ) const = 0; //throw FileError + virtual void removeSymlinkPlain(const AfsPath& linkPath ) const = 0; //throw FileError + virtual void removeFolderPlain (const AfsPath& folderPath) const = 0; //throw FileError //---------------------------------------------------------------------------------------------------------------- - //virtual void setModTime(const AfsPath& afsPath, time_t modTime) const = 0; //throw FileError, follows symlinks + //virtual void setModTime(const AfsPath& itemPath, time_t modTime) const = 0; //throw FileError, follows symlinks - virtual AbstractPath getSymlinkResolvedPath(const AfsPath& afsPath) const = 0; //throw FileError - virtual bool equalSymlinkContentForSameAfsType(const AfsPath& afsLhs, const AbstractPath& apRhs) const = 0; //throw FileError + virtual AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const = 0; //throw FileError + virtual bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const = 0; //throw FileError //---------------------------------------------------------------------------------------------------------------- - virtual std::unique_ptr<InputStream> getInputStream(const AfsPath& afsPath, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) const = 0; //throw FileError, ErrorFileLocked + virtual std::unique_ptr<InputStream> getInputStream(const AfsPath& filePath) const = 0; //throw FileError, ErrorFileLocked //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - virtual std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& afsPath, //throw FileError + virtual std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const zen::IoCallback& notifyUnbufferedIO /*throw X*/) const = 0; + std::optional<time_t> modTime) const = 0; //---------------------------------------------------------------------------------------------------------------- virtual void traverseFolderRecursive(const TraverserWorkload& workload /*throw X*/, size_t parallelOps) const = 0; //---------------------------------------------------------------------------------------------------------------- - virtual bool supportsPermissions(const AfsPath& afsPath) const = 0; //throw FileError + virtual bool supportsPermissions(const AfsPath& folderPath) const = 0; //throw FileError //already existing: undefined behavior! (e.g. fail/overwrite) virtual void moveAndRenameItemForSameAfsType(const AfsPath& pathFrom, const AbstractPath& pathTo) const = 0; //throw FileError, ErrorMoveUnsupported //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - virtual FileCopyResult copyFileForSameAfsType(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, bool copyFilePermissions, + virtual FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, bool copyFilePermissions, //accummulated delta != file size! consider ADS, sparse, compressed files const zen::IoCallback& notifyUnbufferedIO /*throw X*/) const = 0; //symlink handling: follow //already existing: fail - virtual void copyNewFolderForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const = 0; //throw FileError + virtual void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const = 0; //throw FileError //already existing: fail - virtual void copySymlinkForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const = 0; //throw FileError + virtual void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const = 0; //throw FileError //---------------------------------------------------------------------------------------------------------------- - virtual zen::FileIconHolder getFileIcon (const AfsPath& afsPath, int pixelSize) const = 0; //throw FileError; optional return value - virtual zen::ImageHolder getThumbnailImage(const AfsPath& afsPath, int pixelSize) const = 0; //throw FileError; optional return value + virtual zen::FileIconHolder getFileIcon (const AfsPath& filePath, int pixelSize) const = 0; //throw FileError; optional return value + virtual zen::ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const = 0; //throw FileError; optional return value virtual void authenticateAccess(bool allowUserInteraction) const = 0; //throw FileError @@ -406,10 +414,9 @@ private: virtual bool hasNativeTransactionalCopy() const = 0; //---------------------------------------------------------------------------------------------------------------- - virtual int64_t getFreeDiskSpace(const AfsPath& afsPath) const = 0; //throw FileError, returns < 0 if not available - virtual bool supportsRecycleBin(const AfsPath& afsPath) const = 0; //throw FileError - virtual std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& afsPath) const = 0; //throw FileError, return value must be bound! - virtual void recycleItemIfExists(const AfsPath& afsPath) const = 0; //throw FileError + virtual int64_t getFreeDiskSpace(const AfsPath& folderPath) const = 0; //throw FileError, returns < 0 if not available + virtual std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& folderPath) const = 0; //throw FileError, RecycleBinUnavailable + virtual void moveToRecycleBin(const AfsPath& itemPath) const = 0; //throw FileError, RecycleBinUnavailable }; @@ -435,9 +442,9 @@ bool operator==(const AbstractPath& lhs, const AbstractPath& rhs) { return lhs.a //------------------------------------ implementation ----------------------------------------- inline -AbstractPath AbstractFileSystem::appendRelPath(const AbstractPath& ap, const Zstring& relPath) +AbstractPath AbstractFileSystem::appendRelPath(const AbstractPath& itemPath, const Zstring& relPath) { - return AbstractPath(ap.afsDevice, AfsPath(appendPath(ap.afsPath.value, relPath))); + return AbstractPath(itemPath.afsDevice, AfsPath(appendPath(itemPath.afsPath.value, relPath))); } //--------------------------------------------------------------------------------------------- @@ -460,19 +467,21 @@ AbstractFileSystem::OutputStream::~OutputStream() //- also for Native: setFileTime() may fail *after* FileOutput::finalize() try { AbstractFileSystem::removeFilePlain(filePath_); /*throw FileError*/ } catch (zen::FileError&) {} + warn_static("log on error") } inline -void AbstractFileSystem::OutputStream::write(const void* data, size_t len) //throw FileError, X +size_t AbstractFileSystem::OutputStream::tryWrite(const void* buffer, size_t bytesToWrite, const zen::IoCallback& notifyUnbufferedIO /*throw X*/) { - outStream_->write(data, len); //throw FileError, X - bytesWrittenTotal_ += len; + const size_t bytesWritten = outStream_->tryWrite(buffer, bytesToWrite, notifyUnbufferedIO /*throw X*/); //throw FileError, X may return short! + bytesWrittenTotal_ += bytesWritten; + return bytesWritten; } inline -AbstractFileSystem::FinalizeResult AbstractFileSystem::OutputStream::finalize() //throw FileError, X +AbstractFileSystem::FinalizeResult AbstractFileSystem::OutputStream::finalize(const zen::IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { using namespace zen; @@ -483,7 +492,7 @@ AbstractFileSystem::FinalizeResult AbstractFileSystem::OutputStream::finalize() L"%x", formatNumber(*bytesExpected_)), L"%y", formatNumber(bytesWrittenTotal_))); - const FinalizeResult result = outStream_->finalize(); //throw FileError, X + const FinalizeResult result = outStream_->finalize(notifyUnbufferedIO); //throw FileError, X finalizeSucceeded_ = true; return result; } @@ -491,23 +500,23 @@ AbstractFileSystem::FinalizeResult AbstractFileSystem::OutputStream::finalize() //-------------------------------------------------------------------------- inline -bool AbstractFileSystem::supportPermissionCopy(const AbstractPath& apSource, const AbstractPath& apTarget) //throw FileError +bool AbstractFileSystem::supportPermissionCopy(const AbstractPath& sourcePath, const AbstractPath& targetPath) //throw FileError { - if (typeid(apSource.afsDevice.ref()) != typeid(apTarget.afsDevice.ref())) + if (typeid(sourcePath.afsDevice.ref()) != typeid(targetPath.afsDevice.ref())) return false; - return apSource.afsDevice.ref().supportsPermissions(apSource.afsPath) && //throw FileError - apTarget.afsDevice.ref().supportsPermissions(apTarget.afsPath); + return sourcePath.afsDevice.ref().supportsPermissions(sourcePath.afsPath) && //throw FileError + targetPath.afsDevice.ref().supportsPermissions(targetPath.afsPath); } inline -bool AbstractFileSystem::equalSymlinkContent(const AbstractPath& apLhs, const AbstractPath& apRhs) //throw FileError +bool AbstractFileSystem::equalSymlinkContent(const AbstractPath& linkPathL, const AbstractPath& linkPathR) //throw FileError { - if (typeid(apLhs.afsDevice.ref()) != typeid(apRhs.afsDevice.ref())) + if (typeid(linkPathL.afsDevice.ref()) != typeid(linkPathR.afsDevice.ref())) return false; - return apLhs.afsDevice.ref().equalSymlinkContentForSameAfsType(apLhs.afsPath, apRhs); //throw FileError + return linkPathL.afsDevice.ref().equalSymlinkContentForSameAfsType(linkPathL.afsPath, linkPathR); //throw FileError } @@ -527,38 +536,38 @@ void AbstractFileSystem::moveAndRenameItem(const AbstractPath& pathFrom, const A inline -void AbstractFileSystem::copyNewFolder(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions) //throw FileError +void AbstractFileSystem::copyNewFolder(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) //throw FileError { using namespace zen; - if (typeid(apSource.afsDevice.ref()) != typeid(apTarget.afsDevice.ref())) + if (typeid(sourcePath.afsDevice.ref()) != typeid(targetPath.afsDevice.ref())) { //fall back: if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(getDisplayPath(apTarget))), + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(getDisplayPath(targetPath))), _("Operation not supported between different devices.")); //already existing: fail - createFolderPlain(apTarget); //throw FileError + createFolderPlain(targetPath); //throw FileError } else - apSource.afsDevice.ref().copyNewFolderForSameAfsType(apSource.afsPath, apTarget, copyFilePermissions); //throw FileError + sourcePath.afsDevice.ref().copyNewFolderForSameAfsType(sourcePath.afsPath, targetPath, copyFilePermissions); //throw FileError } //already existing: fail inline -void AbstractFileSystem::copySymlink(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions) //throw FileError +void AbstractFileSystem::copySymlink(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) //throw FileError { using namespace zen; - if (typeid(apSource.afsDevice.ref()) != typeid(apTarget.afsDevice.ref())) + if (typeid(sourcePath.afsDevice.ref()) != typeid(targetPath.afsDevice.ref())) throw FileError(replaceCpy(replaceCpy(_("Cannot copy symbolic link %x to %y."), - L"%x", L'\n' + fmtPath(getDisplayPath(apSource))), - L"%y", L'\n' + fmtPath(getDisplayPath(apTarget))), _("Operation not supported between different devices.")); + L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), + L"%y", L'\n' + fmtPath(getDisplayPath(targetPath))), _("Operation not supported between different devices.")); //already existing: fail - apSource.afsDevice.ref().copySymlinkForSameAfsType(apSource.afsPath, apTarget, copyFilePermissions); //throw FileError + sourcePath.afsDevice.ref().copySymlinkForSameAfsType(sourcePath.afsPath, targetPath, copyFilePermissions); //throw FileError } } diff --git a/FreeFileSync/Source/afs/ftp.cpp b/FreeFileSync/Source/afs/ftp.cpp index 41d660d4..ddab97f4 100644 --- a/FreeFileSync/Source/afs/ftp.cpp +++ b/FreeFileSync/Source/afs/ftp.cpp @@ -29,9 +29,11 @@ namespace constexpr std::chrono::seconds FTP_SESSION_MAX_IDLE_TIME (20); constexpr std::chrono::seconds FTP_SESSION_CLEANUP_INTERVAL(4); -const int FTP_STREAM_BUFFER_SIZE = 512 * 1024; //unit: [byte] -//FTP stream buffer should be at least as big as the biggest AFS block size (currently 256 KB for MTP), -//but there seems to be no reason for an upper limit + +const size_t FTP_BLOCK_SIZE_DOWNLOAD = 64 * 1024; //libcurl returns blocks of only 16 kB as returned by recv() even if we request larger blocks via CURLOPT_BUFFERSIZE +const size_t FTP_BLOCK_SIZE_UPLOAD = 64 * 1024; //libcurl requests blocks of 64 kB. larger blocksizes set via CURLOPT_UPLOAD_BUFFERSIZE do not seem to make a difference +const size_t FTP_STREAM_BUFFER_SIZE = 1024 * 1024; //unit: [byte] +//stream buffer should be big enough to facilitate prefetching during alternating read/write operations => e.g. see serialize.h::unbufferedStreamCopy() const Zchar ftpPrefix[] = Zstr("ftp:"); @@ -74,7 +76,7 @@ std::weak_ordering operator<=>(const FtpSessionId& lhs, const FtpSessionId& rhs) namespace { -Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& afsPath); //noexcept +Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& itemPath); //noexcept Zstring ansiToUtfEncoding(const std::string& str) //throw SysError @@ -95,7 +97,7 @@ Zstring ansiToUtfEncoding(const std::string& str) //throw SysError &bytesWritten, //gsize* bytes_written &error); //GError** error if (!utfStr) - throw SysError(formatGlibError("g_convert(" + utfTo<std::string>(str) + ')', error)); + throw SysError(formatGlibError("g_convert(" + utfTo<std::string>(str) + ", LATIN1 -> UTF-8)", error)); ZEN_ON_SCOPE_EXIT(::g_free(utfStr)); return {utfStr, bytesWritten}; @@ -108,20 +110,23 @@ std::string utfToAnsiEncoding(const Zstring& str) //throw SysError { if (str.empty()) return {}; + const Zstring& strNorm = getUnicodeNormalForm(str); //convert to pre-composed *before* attempting conversion + gsize bytesWritten = 0; //not including the terminating null GError* error = nullptr; ZEN_ON_SCOPE_EXIT(if (error) ::g_error_free(error)); - gchar* ansiStr = ::g_convert(str.c_str(), //const gchar* str - str.size(), //gssize len - "LATIN1", //const gchar* to_codeset - "UTF-8", //const gchar* from_codeset - nullptr, //gsize* bytes_read - &bytesWritten, //gsize* bytes_written - &error); //GError** error + //fails for: 1. broken UTF-8 2. not-ANSI-encodable Unicode + gchar* ansiStr = ::g_convert(strNorm.c_str(), //const gchar* str + strNorm.size(), //gssize len + "LATIN1", //const gchar* to_codeset + "UTF-8", //const gchar* from_codeset + nullptr, //gsize* bytes_read + &bytesWritten, //gsize* bytes_written + &error); //GError** error if (!ansiStr) - throw SysError(formatGlibError("g_convert(" + utfTo<std::string>(str) + ')', error)); + throw SysError(formatGlibError("g_convert(" + utfTo<std::string>(strNorm) + ", UTF-8 -> LATIN1)", error)); ZEN_ON_SCOPE_EXIT(::g_free(ansiStr)); return {ansiStr, bytesWritten}; @@ -129,7 +134,7 @@ std::string utfToAnsiEncoding(const Zstring& str) //throw SysError } -std::wstring getCurlDisplayPath(const FtpSessionId& sessionId, const AfsPath& afsPath) +std::wstring getCurlDisplayPath(const FtpSessionId& sessionId, const AfsPath& itemPath) { Zstring displayPath = Zstring(ftpPrefix) + Zstr("//"); @@ -142,7 +147,7 @@ std::wstring getCurlDisplayPath(const FtpSessionId& sessionId, const AfsPath& af port != DEFAULT_PORT_FTP) displayPath += Zstr(':') + numberTo<Zstring>(port); - const Zstring& relPath = getServerRelPath(afsPath); + const Zstring& relPath = getServerRelPath(itemPath); if (relPath != Zstr("/")) displayPath += relPath; @@ -276,7 +281,7 @@ public: void setContextTimeout(const std::weak_ptr<int>& timeoutSec) { timeoutSec_ = timeoutSec; } //returns server response (header data) - std::string perform(const AfsPath& afsPath, bool isDir, curl_ftpmethod pathMethod, + std::string perform(const AfsPath& itemPath, bool isDir, curl_ftpmethod pathMethod, const std::vector<CurlOption>& extraOptions, bool requiresUtf8) //throw SysError { if (requiresUtf8) //avoid endless recursion @@ -297,8 +302,7 @@ public: options.emplace_back(CURLOPT_ERRORBUFFER, curlErrorBuf); std::string headerData; - using CbType = size_t (*)(const char* buffer, size_t size, size_t nitems, void* callbackData); - CbType onHeaderReceived = [](const char* buffer, size_t size, size_t nitems, void* callbackData) + curl_write_callback onHeaderReceived = [](/*const*/ char* buffer, size_t size, size_t nitems, void* callbackData) { auto& output = *static_cast<std::string*>(callbackData); output.append(buffer, size * nitems); @@ -308,7 +312,7 @@ public: options.emplace_back(CURLOPT_HEADERFUNCTION, onHeaderReceived); //lifetime: keep alive until after curl_easy_setopt() below - const std::string curlPath = getCurlUrlPath(afsPath, isDir); //throw SysError + const std::string curlPath = getCurlUrlPath(itemPath, isDir); //throw SysError options.emplace_back(CURLOPT_URL, curlPath.c_str()); assert(pathMethod != CURLFTPMETHOD_MULTICWD); //too slow! @@ -342,7 +346,7 @@ public: options.emplace_back(CURLOPT_LOW_SPEED_LIMIT, 1); //[bytes], can't use "0" which means "inactive", so use some low number //unlike CURLOPT_TIMEOUT, this one is NOT a limit on the total transfer time - options.emplace_back(CURLOPT_FTP_RESPONSE_TIMEOUT, *timeoutSec); //== alias of CURLOPT_SERVER_RESPONSE_TIMEOUT + options.emplace_back(CURLOPT_SERVER_RESPONSE_TIMEOUT, *timeoutSec); //== alias of CURLOPT_SERVER_RESPONSE_TIMEOUT //CURLOPT_ACCEPTTIMEOUT_MS? => only relevant for "active" FTP connections @@ -624,11 +628,11 @@ public: return std::chrono::steady_clock::now() - lastSuccessfulUseTime_ <= FTP_SESSION_MAX_IDLE_TIME; } - std::string getServerPathInternal(const AfsPath& afsPath) //throw SysError + std::string getServerPathInternal(const AfsPath& itemPath) //throw SysError { - const Zstring serverPath = getServerRelPath(afsPath); + const Zstring serverPath = getServerRelPath(itemPath); - if (afsPath.value.empty()) //endless recursion caveat!! utfToServerEncoding() transitively depends on getServerPathInternal() + if (itemPath.value.empty()) //endless recursion caveat!! utfToServerEncoding() transitively depends on getServerPathInternal() return utfTo<std::string>(serverPath); return utfToServerEncoding(serverPath); //throw SysError @@ -696,11 +700,11 @@ private: FtpSession (const FtpSession&) = delete; FtpSession& operator=(const FtpSession&) = delete; - std::string getCurlUrlPath(const AfsPath& afsPath /*optional*/, bool isDir) //throw SysError + std::string getCurlUrlPath(const AfsPath& itemPath /*optional*/, bool isDir) //throw SysError { std::string curlRelPath; //libcurl expects encoded paths (except for '/' char!!!) => bug: https://github.com/curl/curl/pull/4423 - for (const std::string& comp : split(getServerPathInternal(afsPath), '/', SplitOnEmpty::skip)) //throw SysError + for (const std::string& comp : split(getServerPathInternal(itemPath), '/', SplitOnEmpty::skip)) //throw SysError { char* compFmt = ::curl_easy_escape(easyHandle_, comp.c_str(), static_cast<int>(comp.size())); if (!compFmt) @@ -1081,8 +1085,7 @@ public: { std::string rawListing; //get raw FTP directory listing - using CbType = size_t (*)(const char* buffer, size_t size, size_t nitems, void* callbackData); - CbType onBytesReceived = [](const char* buffer, size_t size, size_t nitems, void* callbackData) + curl_write_callback onBytesReceived = [](/*const*/ char* buffer, size_t size, size_t nitems, void* callbackData) { auto& listing = *static_cast<std::string*>(callbackData); listing.append(buffer, size * nitems); @@ -1679,25 +1682,24 @@ void ftpFileDownload(const FtpLogin& login, const AfsPath& afsFilePath, //throw { std::exception_ptr exception; - auto onBytesReceived = [&](const void* buffer, size_t len) + auto onBytesReceived = [&](const void* buffer, size_t bytesToWrite) { try { - writeBlock(buffer, len); //throw X - return len; + writeBlock(buffer, bytesToWrite); //throw X + //[!] let's NOT use "incomplete write Posix semantics" for libcurl! + //who knows if libcurl buffers properly, or if it sends incomplete packages!? + return bytesToWrite; } catch (...) { exception = std::current_exception(); - return len + 1; //signal error condition => CURLE_WRITE_ERROR + return bytesToWrite + 1; //signal error condition => CURLE_WRITE_ERROR } }; - - using CbType = decltype(onBytesReceived); - using CbWrapperType = size_t (*)(const void* buffer, size_t size, size_t nitems, CbType* callbackData); //needed for cdecl function pointer cast - CbWrapperType onBytesReceivedWrapper = [](const void* buffer, size_t size, size_t nitems, CbType* callbackData) + curl_write_callback onBytesReceivedWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { - return (*callbackData)(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda + return (*static_cast<decltype(onBytesReceived)*>(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; try @@ -1709,6 +1711,9 @@ void ftpFileDownload(const FtpLogin& login, const AfsPath& afsFilePath, //throw {CURLOPT_WRITEDATA, &onBytesReceived}, {CURLOPT_WRITEFUNCTION, onBytesReceivedWrapper}, {CURLOPT_IGNORE_CONTENT_LENGTH, 1L}, //skip FTP "SIZE" command before download (=> download until actual EOF if file size changes) + + //{CURLOPT_BUFFERSIZE, 256 * 1024} -> defaults is 16 kB which seems to correspond to SSL packet size + //=> setting larget buffers size does nothing (recv still returns only 16 kB) }, true /*requiresUtf8*/); //throw SysError }); } @@ -1726,18 +1731,22 @@ void ftpFileDownload(const FtpLogin& login, const AfsPath& afsFilePath, //throw freefilesync.org: overwrites FileZilla Server: overwrites Windows IIS: overwrites */ -void ftpFileUpload(const FtpLogin& login, const AfsPath& afsFilePath, //throw FileError, X - const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/) //returning 0 signals EOF: Posix read() semantics +void ftpFileUpload(const FtpLogin& login, const AfsPath& afsFilePath, + const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/) //throw FileError, X; return "bytesToRead" bytes unless end of stream { std::exception_ptr exception; - auto getBytesToSend = [&](void* buffer, size_t len) -> size_t + auto getBytesToSend = [&](void* buffer, size_t bytesToRead) -> size_t { try { - //libcurl calls back until 0 bytes are returned (Posix read() semantics), or, - //if CURLOPT_INFILESIZE_LARGE was set, after exactly this amount of bytes - const size_t bytesRead = readBlock(buffer, len);//throw X; return "bytesToRead" bytes unless end of stream! + /* libcurl calls back until 0 bytes are returned (Posix read() semantics), or, + if CURLOPT_INFILESIZE_LARGE was set, after exactly this amount of bytes + + [!] let's NOT use "incomplete read Posix semantics" for libcurl! + who knows if libcurl buffers properly, or if it requests incomplete packages!? */ + const size_t bytesRead = readBlock(buffer, bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream + assert(bytesRead == bytesToRead || bytesRead == 0 || readBlock(buffer, bytesToRead) == 0); return bytesRead; } catch (...) @@ -1746,33 +1755,31 @@ void ftpFileUpload(const FtpLogin& login, const AfsPath& afsFilePath, //throw Fi return CURL_READFUNC_ABORT; //signal error condition => CURLE_ABORTED_BY_CALLBACK } }; - - using CbType = decltype(getBytesToSend); - using CbWrapperType = size_t (*)(void* buffer, size_t size, size_t nitems, CbType* callbackData); - CbWrapperType getBytesToSendWrapper = [](void* buffer, size_t size, size_t nitems, CbType* callbackData) + curl_read_callback getBytesToSendWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { - return (*callbackData)(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda + return (*static_cast<decltype(getBytesToSend)*>(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; try { accessFtpSession(login, [&](FtpSession& session) //throw SysError { - /* - curl_slist* quote = nullptr; + /* curl_slist* quote = nullptr; ZEN_ON_SCOPE_EXIT(::curl_slist_free_all(quote)); //"prefix the command with an asterisk to make libcurl continue even if the command fails" quote = ::curl_slist_append(quote, ("*DELE " + session.getServerPathInternal(afsFilePath)).c_str()); //throw SysError - //optimize fail-safe copy with RNFR/RNTO as CURLOPT_POSTQUOTE? -> even slightly *slower* than RNFR/RNTO as additional curl_easy_perform() - */ + //optimize fail-safe copy with RNFR/RNTO as CURLOPT_POSTQUOTE? -> even slightly *slower* than RNFR/RNTO as additional curl_easy_perform() */ + session.perform(afsFilePath, false /*isDir*/, CURLFTPMETHOD_NOCWD, //are there any servers that require CURLFTPMETHOD_SINGLECWD? let's find out { {CURLOPT_UPLOAD, 1L}, {CURLOPT_READDATA, &getBytesToSend}, {CURLOPT_READFUNCTION, getBytesToSendWrapper}, + //{CURLOPT_UPLOAD_BUFFERSIZE, 256 * 1024} -> defaults is 64 kB. apparently no performance improvement for larger buffers like 256 kB + //{CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(inputBuffer.size())}, //=> CURLOPT_INFILESIZE_LARGE does not issue a specific FTP command, but is used by libcurl only! @@ -1794,21 +1801,18 @@ void ftpFileUpload(const FtpLogin& login, const AfsPath& afsFilePath, //throw Fi struct InputStreamFtp : public AFS::InputStream { - InputStreamFtp(const FtpLogin& login, - const AfsPath& afsPath, - const IoCallback& notifyUnbufferedIO /*throw X*/) : - notifyUnbufferedIO_(notifyUnbufferedIO) + InputStreamFtp(const FtpLogin& login, const AfsPath& filePath) { - worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, login, afsPath] + worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, login, filePath] { - setCurrentThreadName(Zstr("Istream[FTP] ") + utfTo<Zstring>(getCurlDisplayPath(login, afsPath))); + setCurrentThreadName(Zstr("Istream ") + utfTo<Zstring>(getCurlDisplayPath(login, filePath))); try { auto writeBlock = [&](const void* buffer, size_t bytesToWrite) { - return asyncStreamOut->write(buffer, bytesToWrite); //throw ThreadStopRequest + asyncStreamOut->write(buffer, bytesToWrite); //throw ThreadStopRequest }; - ftpFileDownload(login, afsPath, writeBlock); //throw FileError, ThreadStopRequest + ftpFileDownload(login, filePath, writeBlock); //throw FileError, ThreadStopRequest asyncStreamOut->closeStream(); } @@ -1821,33 +1825,31 @@ struct InputStreamFtp : public AFS::InputStream asyncStreamIn_->setReadError(std::make_exception_ptr(ThreadStopRequest())); } - size_t read(void* buffer, size_t bytesToRead) override //throw FileError, (ErrorFileLocked), X; return "bytesToRead" bytes unless end of stream! + size_t getBlockSize() override { return FTP_BLOCK_SIZE_DOWNLOAD; } //throw (FileError) + + //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + size_t tryRead(void* buffer, size_t bytesToRead, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, (ErrorFileLocked), X { - const size_t bytesRead = asyncStreamIn_->read(buffer, bytesToRead); //throw FileError - reportBytesProcessed(); //throw X + const size_t bytesRead = asyncStreamIn_->tryRead(buffer, bytesToRead); //throw FileError + reportBytesProcessed(notifyUnbufferedIO); //throw X return bytesRead; //no need for asyncStreamIn_->checkWriteErrors(): once end of stream is reached, asyncStreamOut->closeStream() was called => no errors occured } - size_t getBlockSize() const override { return 64 * 1024; } //non-zero block size is AFS contract! - - std::optional<AFS::StreamAttributes> getAttributesBuffered() override //throw FileError - { - return {}; //there is no stream handle => no buffered attribute access! - //PERF: get attributes during file download? - // CURLOPT_FILETIME: test case 77 files, 4MB: overall copy time increases by 12% - // CURLOPT_PREQUOTE/CURLOPT_PREQUOTE/CURLOPT_POSTQUOTE + MDTM: test case 77 files, 4MB: overall copy time increases by 12% - } + std::optional<AFS::StreamAttributes> tryGetAttributesFast() override { return {}; }//throw FileError + //there is no stream handle => no buffered attribute access! + //PERF: get attributes during file download? + // CURLOPT_FILETIME: test case 77 files, 4MB: overall copy time increases by 12% + // CURLOPT_PREQUOTE/CURLOPT_PREQUOTE/CURLOPT_POSTQUOTE + MDTM: test case 77 files, 4MB: overall copy time increases by 12% private: - void reportBytesProcessed() //throw X + void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { - const int64_t totalBytesDownloaded = asyncStreamIn_->getTotalBytesWritten(); - if (notifyUnbufferedIO_) notifyUnbufferedIO_(totalBytesDownloaded - totalBytesReported_); //throw X - totalBytesReported_ = totalBytesDownloaded; + const int64_t bytesDelta = makeSigned(asyncStreamIn_->getTotalBytesWritten()) - totalBytesReported_; + totalBytesReported_ += bytesDelta; + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } - const IoCallback notifyUnbufferedIO_; //throw X int64_t totalBytesReported_ = 0; std::shared_ptr<AsyncStreamBuffer> asyncStreamIn_ = std::make_shared<AsyncStreamBuffer>(FTP_STREAM_BUFFER_SIZE); InterruptibleThread worker_; @@ -1860,30 +1862,27 @@ private: struct OutputStreamFtp : public AFS::OutputStreamImpl { OutputStreamFtp(const FtpLogin& login, - const AfsPath& afsPath, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) : + const AfsPath& filePath, + std::optional<time_t> modTime) : login_(login), - afsPath_(afsPath), - modTime_(modTime), - notifyUnbufferedIO_(notifyUnbufferedIO) + filePath_(filePath), + modTime_(modTime) { std::promise<void> pUploadDone; futUploadDone_ = pUploadDone.get_future(); - worker_ = InterruptibleThread([login, afsPath, + worker_ = InterruptibleThread([login, filePath, asyncStreamIn = this->asyncStreamOut_, pUploadDone = std::move(pUploadDone)]() mutable { - setCurrentThreadName(Zstr("Ostream[FTP] ") + utfTo<Zstring>(getCurlDisplayPath(login, afsPath))); + setCurrentThreadName(Zstr("Ostream ") + utfTo<Zstring>(getCurlDisplayPath(login, filePath))); try { auto readBlock = [&](void* buffer, size_t bytesToRead) { - //returns "bytesToRead" bytes unless end of stream! => maps nicely into Posix read() semantics expected by ftpFileUpload() return asyncStreamIn->read(buffer, bytesToRead); //throw ThreadStopRequest }; - ftpFileUpload(login, afsPath, readBlock); //throw FileError, ThreadStopRequest + ftpFileUpload(login, filePath, readBlock); //throw FileError, ThreadStopRequest assert(asyncStreamIn->getTotalBytesRead() == asyncStreamIn->getTotalBytesWritten()); pUploadDone.set_value(); @@ -1900,29 +1899,37 @@ struct OutputStreamFtp : public AFS::OutputStreamImpl ~OutputStreamFtp() { - asyncStreamOut_->setWriteError(std::make_exception_ptr(ThreadStopRequest())); + if (asyncStreamOut_) //finalize() was not called (successfully) + asyncStreamOut_->setWriteError(std::make_exception_ptr(ThreadStopRequest())); } - void write(const void* buffer, size_t bytesToWrite) override //throw FileError, X + size_t getBlockSize() override { return FTP_BLOCK_SIZE_UPLOAD; } //throw (FileError) + + size_t tryWrite(const void* buffer, size_t bytesToWrite, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 { - asyncStreamOut_->write(buffer, bytesToWrite); //throw FileError - reportBytesProcessed(); //throw X + const size_t bytesWritten = asyncStreamOut_->tryWrite(buffer, bytesToWrite); //throw FileError + reportBytesProcessed(notifyUnbufferedIO); //throw X + return bytesWritten; } - AFS::FinalizeResult finalize() override //throw FileError, X + AFS::FinalizeResult finalize(const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X { + if (!asyncStreamOut_) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + asyncStreamOut_->closeStream(); while (futUploadDone_.wait_for(std::chrono::milliseconds(50)) == std::future_status::timeout) - reportBytesProcessed(); //throw X - reportBytesProcessed(); //[!] once more, now that *all* bytes were written - - asyncStreamOut_->checkReadErrors(); //throw FileError - //-------------------------------------------------------------------- + reportBytesProcessed(notifyUnbufferedIO); //throw X + reportBytesProcessed(notifyUnbufferedIO); //[!] once more, now that *all* bytes were written assert(isReady(futUploadDone_)); futUploadDone_.get(); //throw FileError + //asyncStreamOut_->checkReadErrors(); //throw FileError -> not needed after *successful* upload + asyncStreamOut_.reset(); //do NOT reset on failure, so that ~OutputStreamFtp() will request worker thread to stop + //-------------------------------------------------------------------- + AFS::FinalizeResult result; //result.filePrint = ... -> yet unknown at this point try @@ -1937,11 +1944,11 @@ struct OutputStreamFtp : public AFS::OutputStreamImpl } private: - void reportBytesProcessed() //throw X + void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { - const int64_t totalBytesUploaded = asyncStreamOut_->getTotalBytesRead(); - if (notifyUnbufferedIO_) notifyUnbufferedIO_(totalBytesUploaded - totalBytesReported_); //throw X - totalBytesReported_ = totalBytesUploaded; + const int64_t bytesDelta = makeSigned(asyncStreamOut_->getTotalBytesRead()) - totalBytesReported_; + totalBytesReported_ += bytesDelta; + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } void setModTimeIfAvailable() const //throw FileError, follows symlinks @@ -1959,21 +1966,20 @@ private: if (!session.supportsMfmt()) //throw SysError throw SysError(L"Server does not support the MFMT command."); - session.runSingleFtpCommand("MFMT " + isoTime + ' ' + session.getServerPathInternal(afsPath_), + session.runSingleFtpCommand("MFMT " + isoTime + ' ' + session.getServerPathInternal(filePath_), true /*requiresUtf8*/); //throw SysError //not relevant for OutputStreamFtp, but: does MFMT follow symlinks? for Linux FTP server (using utime) it does }); } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(getCurlDisplayPath(login_, afsPath_))), e.toString()); + throw FileError(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(getCurlDisplayPath(login_, filePath_))), e.toString()); } } const FtpLogin login_; - const AfsPath afsPath_; + const AfsPath filePath_; const std::optional<time_t> modTime_; - const IoCallback notifyUnbufferedIO_; //throw X int64_t totalBytesReported_ = 0; std::shared_ptr<AsyncStreamBuffer> asyncStreamOut_ = std::make_shared<AsyncStreamBuffer>(FTP_STREAM_BUFFER_SIZE); InterruptibleThread worker_; @@ -1991,11 +1997,11 @@ public: const FtpLogin& getLogin() const { return login_; } private: - Zstring getInitPathPhrase(const AfsPath& afsPath) const override { return concatenateFtpFolderPathPhrase(login_, afsPath); } + Zstring getInitPathPhrase(const AfsPath& itemPath) const override { return concatenateFtpFolderPathPhrase(login_, itemPath); } - std::vector<Zstring> getPathPhraseAliases(const AfsPath& afsPath) const override { return {getInitPathPhrase(afsPath)}; } + std::vector<Zstring> getPathPhraseAliases(const AfsPath& itemPath) const override { return {getInitPathPhrase(itemPath)}; } - std::wstring getDisplayPath(const AfsPath& afsPath) const override { return getCurlDisplayPath(login_, afsPath); } + std::wstring getDisplayPath(const AfsPath& itemPath) const override { return getCurlDisplayPath(login_, itemPath); } bool isNullFileSystem() const override { return login_.server.empty(); } @@ -2019,11 +2025,11 @@ private: } //---------------------------------------------------------------------------------------------------------------- - ItemType getItemType(const AfsPath& afsPath) const override //throw FileError + ItemType getItemType(const AfsPath& itemPath) const override //throw FileError { //don't use MLST: broken for Pure-FTPd: https://freefilesync.org/forum/viewtopic.php?t=4287 - const std::optional<AfsPath> parentAfsPath = getParentPath(afsPath); + const std::optional<AfsPath> parentAfsPath = getParentPath(itemPath); if (!parentAfsPath) //device root => do a quick access tests to see if the server responds at all! try { @@ -2033,9 +2039,9 @@ private: }); return ItemType::folder; } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(getCurlDisplayPath(login_, afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(getCurlDisplayPath(login_, itemPath))), e.toString()); } - const Zstring itemName = getItemName(afsPath); + const Zstring itemName = getItemName(itemPath); assert(!itemName.empty()); try { @@ -2048,16 +2054,16 @@ private: } catch (const ItemType& type) { return type; } //yes, exceptions for control-flow are bad design... but, but... - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(afsPath))), L"File not found."); + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(itemPath))), L"File not found."); } - std::optional<ItemType> itemStillExists(const AfsPath& afsPath) const override //throw FileError + std::optional<ItemType> itemStillExists(const AfsPath& itemPath) const override //throw FileError { - const std::optional<AfsPath> parentAfsPath = getParentPath(afsPath); + const std::optional<AfsPath> parentAfsPath = getParentPath(itemPath); if (!parentAfsPath) //device root - return getItemType(afsPath); //throw FileError; do a simple access test + return getItemType(itemPath); //throw FileError; do a simple access test - const Zstring itemName = getItemName(afsPath); + const Zstring itemName = getItemName(itemPath); assert(!itemName.empty()); try { @@ -2082,46 +2088,46 @@ private: // freefilesync.org: "550 Can't create directory: File exists" // FileZilla Server: "550 Directory already exists" // Windows IIS: "550 Cannot create a file when that file already exists" - void createFolderPlain(const AfsPath& afsPath) const override //throw FileError + void createFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { - session.runSingleFtpCommand("MKD " + session.getServerPathInternal(afsPath), + session.runSingleFtpCommand("MKD " + session.getServerPathInternal(folderPath), true /*requiresUtf8*/); //throw SysError }); } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeFilePlain(const AfsPath& afsPath) const override //throw FileError + void removeFilePlain(const AfsPath& filePath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { - session.runSingleFtpCommand("DELE " + session.getServerPathInternal(afsPath), + session.runSingleFtpCommand("DELE " + session.getServerPathInternal(filePath), true /*requiresUtf8*/); //throw SysError }); } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } - void removeSymlinkPlain(const AfsPath& afsPath) const override //throw FileError + void removeSymlinkPlain(const AfsPath& linkPath) const override //throw FileError { - this->removeFilePlain(afsPath); //throw FileError + this->removeFilePlain(linkPath); //throw FileError //works fine for Linux hosts, but what about Windows-hosted FTP??? Distinguish DELE/RMD? //Windows test, FileZilla Server and Windows IIS FTP: all symlinks are reported as regular folders } - void removeFolderPlain(const AfsPath& afsPath) const override //throw FileError + void removeFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { @@ -2131,7 +2137,7 @@ private: { try { - session.runSingleFtpCommand("RMD " + session.getServerPathInternal(afsPath), + session.runSingleFtpCommand("RMD " + session.getServerPathInternal(folderPath), true /*requiresUtf8*/); //throw SysError } catch (const SysError& e) { delError = e; } @@ -2142,59 +2148,58 @@ private: //Windows test, FileZilla Server and Windows IIS FTP: all symlinks are reported as regular folders //tested freefilesync.org: RMD will fail for symlinks! bool symlinkExists = false; - try { symlinkExists = getItemType(afsPath) == ItemType::symlink; } /*throw FileError*/ catch (FileError&) {} //previous exception is more relevant + try { symlinkExists = getItemType(folderPath) == ItemType::symlink; } /*throw FileError*/ catch (FileError&) {} //previous exception is more relevant if (symlinkExists) - return removeSymlinkPlain(afsPath); //throw FileError + return removeSymlinkPlain(folderPath); //throw FileError else throw* delError; } } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError + void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const override //one call for each object! { //default implementation: folder traversal - AFS::removeFolderIfExistsRecursion(afsPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X + AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X } //---------------------------------------------------------------------------------------------------------------- - AbstractPath getSymlinkResolvedPath(const AfsPath& afsPath) const override //throw FileError + AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const override //throw FileError { - throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(afsPath))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(linkPath))), _("Operation not supported by device.")); } - bool equalSymlinkContentForSameAfsType(const AfsPath& afsLhs, const AbstractPath& apRhs) const override //throw FileError + bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const override //throw FileError { - throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(getDisplayPath(afsLhs))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(getDisplayPath(linkPathL))), _("Operation not supported by device.")); } //---------------------------------------------------------------------------------------------------------------- //return value always bound: - std::unique_ptr<InputStream> getInputStream(const AfsPath& afsPath, const IoCallback& notifyUnbufferedIO /*throw X*/) const override //throw FileError, (ErrorFileLocked) + std::unique_ptr<InputStream> getInputStream(const AfsPath& filePath) const override //throw FileError, (ErrorFileLocked) { - return std::make_unique<InputStreamFtp>(login_, afsPath, notifyUnbufferedIO); + return std::make_unique<InputStreamFtp>(login_, filePath); } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: fail(+delete!)/overwrite/auto-rename - std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& afsPath, //throw FileError + std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) const override + std::optional<time_t> modTime) const override { /* most FTP servers overwrite, but some (e.g. IIS) can be configured to fail, others (pureFTP) can be configured to auto-rename: https://download.pureftpd.org/pub/pure-ftpd/doc/README '-r': Never overwrite existing files. Uploading a file whose name already exists causes an automatic rename. Files are called xyz, xyz.1, xyz.2, xyz.3, etc. */ //already existing: fail (+ delete!!!) - return std::make_unique<OutputStreamFtp>(login_, afsPath, modTime, notifyUnbufferedIO); + return std::make_unique<OutputStreamFtp>(login_, filePath, modTime); } //---------------------------------------------------------------------------------------------------------------- @@ -2206,34 +2211,34 @@ private: //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - FileCopyResult copyFileForSameAfsType(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), X - const AbstractPath& apTarget, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override + FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), X + const AbstractPath& targetPath, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override { //no native FTP file copy => use stream-based file copy: if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - return copyFileAsStream(afsSource, attrSource, apTarget, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X + return copyFileAsStream(sourcePath, attrSource, targetPath, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X } //symlink handling: follow //already existing: fail - void copyNewFolderForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: fail - AFS::createFolderPlain(apTarget); //throw FileError + AFS::createFolderPlain(targetPath); //throw FileError } //already existing: fail - void copySymlinkForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override + void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override { throw FileError(replaceCpy(replaceCpy(_("Cannot copy symbolic link %x to %y."), - L"%x", L'\n' + fmtPath(getDisplayPath(afsSource))), - L"%y", L'\n' + fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), + L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); } //already existing: undefined behavior! (e.g. fail/overwrite) @@ -2273,12 +2278,12 @@ private: } } - bool supportsPermissions(const AfsPath& afsPath) const override { return false; } //throw FileError + bool supportsPermissions(const AfsPath& folderPath) const override { return false; } //throw FileError //wait until there is real demand for copying from and to FTP with permissions => use stream-based file copy: //---------------------------------------------------------------------------------------------------------------- - FileIconHolder getFileIcon (const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value - ImageHolder getThumbnailImage(const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value + FileIconHolder getFileIcon (const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value + ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value void authenticateAccess(bool allowUserInteraction) const override {} //throw FileError @@ -2287,20 +2292,18 @@ private: bool hasNativeTransactionalCopy() const override { return false; } //---------------------------------------------------------------------------------------------------------------- - int64_t getFreeDiskSpace(const AfsPath& afsPath) const override { return -1; } //throw FileError, returns < 0 if not available - - bool supportsRecycleBin(const AfsPath& afsPath) const override { return false; } //throw FileError + int64_t getFreeDiskSpace(const AfsPath& folderPath) const override { return -1; } //throw FileError, returns < 0 if not available - std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& afsPath) const override //throw FileError, return value must be bound! + std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& folderPath) const override //throw FileError, RecycleBinUnavailable { - assert(false); //see supportsRecycleBin() - throw FileError(L"Recycle bin not supported by device."); + throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(folderPath))), + _("Operation not supported by device.")); } - void recycleItemIfExists(const AfsPath& afsPath) const override //throw FileError + void moveToRecycleBin(const AfsPath& itemPath) const override //throw FileError, RecycleBinUnavailable { - assert(false); //see supportsRecycleBin() - throw FileError(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(getDisplayPath(afsPath))), _("Operation not supported by device.")); + throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(itemPath))), + _("Operation not supported by device.")); } const FtpLogin login_; @@ -2309,7 +2312,7 @@ private: //=========================================================================================================================== //expects "clean" login data -Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& afsPath) //noexcept +Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& folderPath) //noexcept { Zstring username; if (!login.username.empty()) @@ -2319,7 +2322,7 @@ Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& afs if (login.port > 0) port = Zstr(':') + numberTo<Zstring>(login.port); - Zstring relPath = getServerRelPath(afsPath); + Zstring relPath = getServerRelPath(folderPath); if (relPath == Zstr("/")) relPath.clear(); diff --git a/FreeFileSync/Source/afs/ftp_common.h b/FreeFileSync/Source/afs/ftp_common.h index 233edd26..c843a943 100644 --- a/FreeFileSync/Source/afs/ftp_common.h +++ b/FreeFileSync/Source/afs/ftp_common.h @@ -56,13 +56,13 @@ Zstring decodeFtpUsername(Zstring name) //(S)FTP path relative to server root using Unix path separators and with leading slash inline -Zstring getServerRelPath(const AfsPath& afsPath) +Zstring getServerRelPath(const AfsPath& itemPath) { using namespace zen; if constexpr (FILE_NAME_SEPARATOR != Zstr('/' )) - return Zstr('/') + replaceCpy(afsPath.value, FILE_NAME_SEPARATOR, Zstr('/')); + return Zstr('/') + replaceCpy(itemPath.value, FILE_NAME_SEPARATOR, Zstr('/')); else - return Zstr('/') + afsPath.value; + return Zstr('/') + itemPath.value; } } diff --git a/FreeFileSync/Source/afs/gdrive.cpp b/FreeFileSync/Source/afs/gdrive.cpp index a220f340..97177810 100644 --- a/FreeFileSync/Source/afs/gdrive.cpp +++ b/FreeFileSync/Source/afs/gdrive.cpp @@ -77,7 +77,10 @@ constexpr std::chrono::seconds HTTP_SESSION_MAX_IDLE_TIME (20); constexpr std::chrono::seconds HTTP_SESSION_CLEANUP_INTERVAL(4); constexpr std::chrono::seconds GDRIVE_SYNC_INTERVAL (5); -const int GDRIVE_STREAM_BUFFER_SIZE = 512 * 1024; //unit: [byte] +const size_t GDRIVE_BLOCK_SIZE_DOWNLOAD = 64 * 1024; //libcurl returns blocks of only 16 kB as returned by recv() even if we request larger blocks via CURLOPT_BUFFERSIZE +const size_t GDRIVE_BLOCK_SIZE_UPLOAD = 64 * 1024; //libcurl requests blocks of 64 kB. larger blocksizes set via CURLOPT_UPLOAD_BUFFERSIZE do not seem to make a difference +const size_t GDRIVE_STREAM_BUFFER_SIZE = 1024 * 1024; //unit: [byte] +//stream buffer should be big enough to facilitate prefetching during alternating read/write operations => e.g. see serialize.h::unbufferedStreamCopy() const Zchar gdrivePrefix[] = Zstr("gdrive:"); const char gdriveFolderMimeType [] = "application/vnd.google-apps.folder"; @@ -312,8 +315,8 @@ HttpSession::Result googleHttpsRequest(const Zstring& serverName, const std::str const std::vector<std::string>& extraHeaders, std::vector<CurlOption> extraOptions, const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, //optional - const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; returning 0 signals EOF - const std::function<void (const std::string_view& header)>& receiveHeader /*throw X*/, //optional + const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; return "bytesToRead" bytes unless end of stream! + const std::function<void (const std::string_view& header)>& receiveHeader /*throw X*/, //optional int timeoutSec) { //https://developers.google.com/drive/api/v3/performance @@ -339,7 +342,7 @@ HttpSession::Result gdriveHttpsRequest(const std::string& serverRelPath, //throw std::vector<std::string> extraHeaders, const std::vector<CurlOption>& extraOptions, const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, //optional - const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; returning 0 signals EOF + const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; return "bytesToRead" bytes unless end of stream! const std::function<void (const std::string_view& header)>& receiveHeader /*throw X*/, //optional const GdriveAccess& access) { @@ -349,7 +352,7 @@ HttpSession::Result gdriveHttpsRequest(const std::string& serverRelPath, //throw extraHeaders, extraOptions, writeResponse /*throw X*/, - readRequest /*throw X*/, + readRequest /*throw X*/, receiveHeader /*throw X*/, access.timeoutSec); //throw SysError, X } @@ -475,6 +478,7 @@ GdriveAccessInfo gdriveAuthorizeAccess(const std::string& gdriveLoginHint, const if (testSocket == invalidSocket) THROW_LAST_SYS_ERROR_WSA("socket"); ZEN_ON_SCOPE_FAIL(closeSocket(testSocket)); + warn_static("log on error!") if (::bind(testSocket, ai.ai_addr, static_cast<int>(ai.ai_addrlen)) != 0) THROW_LAST_SYS_ERROR_WSA("bind"); @@ -1509,25 +1513,28 @@ void gdriveDownloadFileImpl(const std::string& fileId, const std::function<void( if (acknowledgeAbuse) //apply on demand only! https://freefilesync.org/forum/viewtopic.php?t=7520") queryParams += '&' + xWwwFormUrlEncode({{"acknowledgeAbuse", "true"}}); - std::string responseHead; //save front part of the response in case we get an error - bool headFlushed = false; + std::string headBytes; + bool headBytesWritten = false; const HttpSession::Result httpResult = gdriveHttpsRequest("/drive/v3/files/" + fileId + '?' + queryParams, {} /*extraHeaders*/, {} /*extraOptions*/, [&](std::span<const char> buf) + /* libcurl feeds us a shitload of tiny kB-sized zlib-decompressed pieces of data! + libcurls zlib buffer is sized at ridiculous 16 kB! + => if this ever becomes a perf issue: roll our own zlib decompression! */ { - if (responseHead.size() < 10000) //don't access writeBlock() in case of error! (=> support acknowledgeAbuse retry handling) - responseHead.append(buf.data(), buf.size()); + if (headBytes.size() < 16 * 1024) //don't access writeBlock() yet in case of error! (=> support acknowledgeAbuse retry handling) + headBytes.append(buf.data(), buf.size()); else { - if (!headFlushed) + if (!headBytesWritten) { - headFlushed = true; - writeBlock(responseHead.c_str(), responseHead.size()); //throw X + headBytesWritten = true; + writeBlock(headBytes.c_str(), headBytes.size()); //throw X } writeBlock(buf.data(), buf.size()); //throw X } - }, nullptr /*readRequest*/, nullptr /*receiveHeader*/, access); //throw SysError, X + }, nullptr /*tryReadRequest*/, nullptr /*receiveHeader*/, access); //throw SysError, X if (httpResult.statusCode / 100 != 2) { @@ -1536,16 +1543,15 @@ void gdriveDownloadFileImpl(const std::string& fileId, const std::function<void( "reason": "cannotDownloadAbusiveFile", "message": "This file has been identified as malware or spam and cannot be downloaded." }], "code": 403, - "message": "This file has been identified as malware or spam and cannot be downloaded." }} - */ - if (!headFlushed && httpResult.statusCode == 403 && contains(responseHead, "\"cannotDownloadAbusiveFile\"")) - throw SysErrorAbusiveFile(formatGdriveErrorRaw(responseHead)); + "message": "This file has been identified as malware or spam and cannot be downloaded." }} */ + if (!headBytesWritten && httpResult.statusCode == 403 && contains(headBytes, "\"cannotDownloadAbusiveFile\"")) + throw SysErrorAbusiveFile(formatGdriveErrorRaw(headBytes)); - throw SysError(formatGdriveErrorRaw(responseHead)); + throw SysError(formatGdriveErrorRaw(headBytes)); } - if (!headFlushed) - writeBlock(responseHead.c_str(), responseHead.size()); //throw X + if (!headBytesWritten) + writeBlock(headBytes.c_str(), headBytes.size()); //throw X } @@ -1568,7 +1574,7 @@ void gdriveDownloadFile(const std::string& fileId, const std::function<void(cons //note: Google Drive upload is already transactional! //upload "small files" (5 MB or less; enforced by Google?) in a single round-trip std::string /*itemId*/ gdriveUploadSmallFile(const Zstring& fileName, const std::string& parentId, uint64_t streamSize, std::optional<time_t> modTime, //throw SysError, X - const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/, //returning 0 signals EOF: Posix read() semantics + const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X; return "bytesToRead" bytes unless end of stream*/, const GdriveAccess& access) { //https://developers.google.com/drive/api/v3/folder#inserting_a_file_in_a_folder @@ -1602,38 +1608,39 @@ std::string /*itemId*/ gdriveUploadSmallFile(const Zstring& fileName, const std: auto readMultipartBlock = [&, headPos = size_t(0), eof = false, tailPos = size_t(0)](void* buffer, size_t bytesToRead) mutable -> size_t { - auto it = static_cast<std::byte*>(buffer); - const auto itEnd = it + bytesToRead; + const auto bufStart = buffer; if (headPos < postBufHead.size()) { - const size_t junkSize = std::min<ptrdiff_t>(postBufHead.size() - headPos, itEnd - it); - std::memcpy(it, &postBufHead[headPos], junkSize); + const size_t junkSize = std::min<ptrdiff_t>(postBufHead.size() - headPos, bytesToRead); + std::memcpy(buffer, postBufHead.c_str() + headPos, junkSize); headPos += junkSize; - it += junkSize; + buffer = static_cast<std::byte*>(buffer) + junkSize; + bytesToRead -= junkSize; } - if (it != itEnd) + if (bytesToRead > 0) { if (!eof) //don't assume readBlock() will return streamSize bytes as promised => exhaust and let Google Drive fail if there is a mismatch in Content-Length! { - const size_t junkSize = readBlock(it, itEnd - it); //throw X - it += junkSize; + const size_t bytesRead = readBlock(buffer, bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream + buffer = static_cast<std::byte*>(buffer) + bytesRead; + bytesToRead -= bytesRead; - if (junkSize != 0) - return it - static_cast<std::byte*>(buffer); //perf: if input stream is at the end, should we immediately append postBufTail (and avoid extra TCP package)? => negligible! - else + if (bytesToRead > 0) eof = true; } - if (it != itEnd) + if (bytesToRead > 0) if (tailPos < postBufTail.size()) { - const size_t junkSize = std::min<ptrdiff_t>(postBufTail.size() - tailPos, itEnd - it); - std::memcpy(it, &postBufTail[tailPos], junkSize); + const size_t junkSize = std::min<ptrdiff_t>(postBufTail.size() - tailPos, bytesToRead); + std::memcpy(buffer, postBufTail.c_str() + tailPos, junkSize); tailPos += junkSize; - it += junkSize; + buffer = static_cast<std::byte*>(buffer) + junkSize; + bytesToRead -= junkSize; } } - return it - static_cast<std::byte*>(buffer); + return static_cast<std::byte*>(buffer) - + static_cast<std::byte*>(bufStart); }; TODO: @@ -1651,8 +1658,8 @@ TODO: "Content-Length: " + numberTo<std::string>(postBufHead.size() + streamSize + postBufTail.size()) }, {{CURLOPT_POST, 1}}, //otherwise HttpSession::perform() will PUT - [&](std::span<const char> buf) { response.append(buf.data(), buf.size()); }, readMultipartBlock, - nullptr /*receiveHeader*/, access); //throw SysError, X + [&](std::span<const char> buf) { response.append(buf.data(), buf.size()); }, + readMultipartBlock, nullptr /*receiveHeader*/, access); //throw SysError, X JsonValue jresponse; try { jresponse = parseJson(response); } @@ -1670,7 +1677,7 @@ TODO: //file name already existing? => duplicate file created! //note: Google Drive upload is already transactional! std::string /*itemId*/ gdriveUploadFile(const Zstring& fileName, const std::string& parentId, std::optional<time_t> modTime, //throw SysError, X - const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/, //returning 0 signals EOF: Posix read() semantics + const std::function<size_t(void* buffer, size_t bytesToRead)>& tryReadBlock /*throw X*/, //returning 0 signals EOF: Posix read() semantics const GdriveAccess& access) { //https://developers.google.com/drive/api/v3/folder#inserting_a_file_in_a_folder @@ -1729,14 +1736,13 @@ std::string /*itemId*/ gdriveUploadFile(const Zstring& fileName, const std::stri //step 2: upload file content //not officially documented, but Google Drive supports compressed file upload when "Content-Encoding: gzip" is set! :))) - InputStreamAsGzip gzipStream(readBlock); //throw SysError + InputStreamAsGzip gzipStream(tryReadBlock, GDRIVE_BLOCK_SIZE_UPLOAD); //throw SysError - auto readBlockAsGzip = [&](std::span<char> buf) { return gzipStream.read(buf.data(), buf.size()); }; //throw SysError, X - //returns "bytesToRead" bytes unless end of stream! => fits into "0 signals EOF: Posix read() semantics" + auto readRequest = [&](std::span<char> buf) { return gzipStream.read(buf.data(), buf.size()); }; //throw SysError, X std::string response; //don't need "Authorization: Bearer": googleHttpsRequest(GOOGLE_REST_API_SERVER, uploadUrlRelative, { "Content-Encoding: gzip" }, {} /*extraOptions*/, - [&](std::span<const char> buf) { response.append(buf.data(), buf.size()); }, readBlockAsGzip, + [&](std::span<const char> buf) { response.append(buf.data(), buf.size()); }, readRequest, nullptr /*receiveHeader*/, access.timeoutSec); //throw SysError, X JsonValue jresponse; @@ -1758,7 +1764,7 @@ public: explicit GdriveAccessBuffer(const GdriveAccessInfo& accessInfo) : accessInfo_(accessInfo) {} - GdriveAccessBuffer(MemoryStreamIn<std::string>& stream) //throw SysError + GdriveAccessBuffer(MemoryStreamIn& stream) //throw SysError { accessInfo_.accessToken.validUntil = readNumber<int64_t>(stream); // accessInfo_.accessToken.value = readContainer<std::string>(stream); // @@ -1767,7 +1773,7 @@ public: accessInfo_.userInfo.email = readContainer<std::string>(stream); // } - void serialize(MemoryStreamOut<std::string>& stream) const + void serialize(MemoryStreamOut& stream) const { writeNumber<int64_t>(stream, accessInfo_.accessToken.validUntil); static_assert(sizeof(accessInfo_.accessToken.validUntil) <= sizeof(int64_t)); //ensure cross-platform compatibility! @@ -1842,7 +1848,7 @@ public: sharedDriveName_(sharedDriveName), accessBuf_(accessBuf) { assert(!driveId.empty() && sharedDriveName != Zstr("My Drive")); } - GdriveFileState(MemoryStreamIn<std::string>& stream, GdriveAccessBuffer& accessBuf) : //throw SysError + GdriveFileState(MemoryStreamIn& stream, GdriveAccessBuffer& accessBuf) : //throw SysError accessBuf_(accessBuf) { lastSyncToken_ = readContainer<std::string>(stream); // @@ -1879,7 +1885,7 @@ public: } } - void serialize(MemoryStreamOut<std::string>& stream) const + void serialize(MemoryStreamOut& stream) const { writeContainer(stream, lastSyncToken_); writeContainer(stream, driveId_); @@ -1944,28 +1950,28 @@ public: AfsPath existingPath; //input path =: existingPath + relPath std::vector<Zstring> relPath; // }; - PathStatus getPathStatus(const std::string& locationRootId, const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + PathStatus getPathStatus(const std::string& locationRootId, const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - const std::vector<Zstring> relPath = split(afsPath.value, FILE_NAME_SEPARATOR, SplitOnEmpty::skip); + const std::vector<Zstring> relPath = split(itemPath.value, FILE_NAME_SEPARATOR, SplitOnEmpty::skip); if (relPath.empty()) return {locationRootId, GdriveItemType::folder, AfsPath(), {}}; else return getPathStatusSub(locationRootId, AfsPath(), relPath, followLeafShortcut); //throw SysError } - std::string /*itemId*/ getItemId(const std::string& locationRootId, const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + std::string /*itemId*/ getItemId(const std::string& locationRootId, const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - const GdriveFileState::PathStatus& ps = getPathStatus(locationRootId, afsPath, followLeafShortcut); //throw SysError + const GdriveFileState::PathStatus& ps = getPathStatus(locationRootId, itemPath, followLeafShortcut); //throw SysError if (ps.relPath.empty()) return ps.existingItemId; - const AfsPath afsPathMissingChild(appendPath(ps.existingPath.value, ps.relPath.front())); - throw SysError(replaceCpy(_("Cannot find %x."), L"%x", fmtPath(getShortDisplayPath(afsPathMissingChild)))); + const AfsPath itemPathMissingChild(appendPath(ps.existingPath.value, ps.relPath.front())); + throw SysError(replaceCpy(_("Cannot find %x."), L"%x", fmtPath(getShortDisplayPath(itemPathMissingChild)))); } - std::pair<std::string /*itemId*/, GdriveItemDetails> getFileAttributes(const std::string& locationRootId, const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + std::pair<std::string /*itemId*/, GdriveItemDetails> getFileAttributes(const std::string& locationRootId, const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - if (afsPath.value.empty()) //location root not covered by itemDetails_ + if (itemPath.value.empty()) //location root not covered by itemDetails_ { GdriveItemDetails rootDetails { @@ -1976,7 +1982,7 @@ public: return {locationRootId, std::move(rootDetails)}; } - const std::string itemId = getItemId(locationRootId, afsPath, followLeafShortcut); //throw SysError + const std::string itemId = getItemId(locationRootId, itemPath, followLeafShortcut); //throw SysError if (auto it = itemDetails_.find(itemId); it != itemDetails_.end()) return *it; @@ -2118,9 +2124,9 @@ private: friend class GdriveDrivesBuffer; - std::wstring getShortDisplayPath(const AfsPath& afsPath) const + std::wstring getShortDisplayPath(const AfsPath& itemPath) const { - return utfTo<std::wstring>(FILE_NAME_SEPARATOR + afsPath.value); //sufficient info for SysError + we don't have a locationName anyway + return utfTo<std::wstring>(FILE_NAME_SEPARATOR + itemPath.value); //sufficient info for SysError + we don't have a locationName anyway } void notifyItemUpdated(const FileStateDelta& stateDelta, const std::string& itemId, const GdriveItemDetails* details) @@ -2350,19 +2356,19 @@ class GdriveFileStateAtLocation public: GdriveFileStateAtLocation(GdriveFileState& fileState, const std::string& locationRootId) : fileState_(fileState), locationRootId_(locationRootId) {} - GdriveFileState::PathStatus getPathStatus(const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + GdriveFileState::PathStatus getPathStatus(const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - return fileState_.getPathStatus(locationRootId_, afsPath, followLeafShortcut); //throw SysError + return fileState_.getPathStatus(locationRootId_, itemPath, followLeafShortcut); //throw SysError } - std::string /*itemId*/ getItemId(const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + std::string /*itemId*/ getItemId(const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - return fileState_.getItemId(locationRootId_, afsPath, followLeafShortcut); //throw SysError + return fileState_.getItemId(locationRootId_, itemPath, followLeafShortcut); //throw SysError } - std::pair<std::string /*itemId*/, GdriveItemDetails> getFileAttributes(const AfsPath& afsPath, bool followLeafShortcut) //throw SysError + std::pair<std::string /*itemId*/, GdriveItemDetails> getFileAttributes(const AfsPath& itemPath, bool followLeafShortcut) //throw SysError { - return fileState_.getFileAttributes(locationRootId_, afsPath, followLeafShortcut); //throw SysError + return fileState_.getFileAttributes(locationRootId_, itemPath, followLeafShortcut); //throw SysError } GdriveFileState& all() { return fileState_; } @@ -2380,7 +2386,7 @@ public: accessBuf_(accessBuf), myDrive_(getMyDriveId(accessBuf.getAccessToken()), Zstring() /*sharedDriveName*/, accessBuf) {} //throw SysError - GdriveDrivesBuffer(MemoryStreamIn<std::string>& stream, GdriveAccessBuffer& accessBuf) : //throw SysError + GdriveDrivesBuffer(MemoryStreamIn& stream, GdriveAccessBuffer& accessBuf) : //throw SysError accessBuf_(accessBuf), myDrive_(stream, accessBuf) //throw SysError { @@ -2392,7 +2398,7 @@ public: } } - void serialize(MemoryStreamOut<std::string>& stream) const + void serialize(MemoryStreamOut& stream) const { myDrive_.serialize(stream); @@ -2755,11 +2761,11 @@ private: static void saveSession(const Zstring& dbFilePath, const UserSession& userSession) //throw FileError { - MemoryStreamOut<std::string> streamOut; + MemoryStreamOut streamOut; writeArray(streamOut, DB_FILE_DESCR, sizeof(DB_FILE_DESCR)); writeNumber<int32_t>(streamOut, DB_FILE_VERSION); - MemoryStreamOut<std::string> streamOutBody; + MemoryStreamOut streamOutBody; userSession.accessBuf.ref().serialize(streamOutBody); userSession.drivesBuf.ref().serialize(streamOutBody); @@ -2799,7 +2805,8 @@ private: //TODO: remove migration code at some time! 2020-07-03 if (!std::equal(std::begin(tmp), std::end(tmp), std::begin(DB_FILE_DESCR))) { - MemoryStreamIn streamIn2(decompress(byteStream)); //throw SysError + const std::string& uncompressedStream = decompress(byteStream); //throw SysError + MemoryStreamIn streamIn2(uncompressedStream); //-------- file format header -------- const char DB_FILE_DESCR_OLD[] = "FreeFileSync: Google Drive Database"; char tmp2[sizeof(DB_FILE_DESCR_OLD)] = {}; @@ -2831,7 +2838,8 @@ private: version != DB_FILE_VERSION) throw SysError(_("Unsupported data format.") + L' ' + replaceCpy(_("Version: %x"), L"%x", numberTo<std::wstring>(version))); - MemoryStreamIn streamInBody(decompress(std::string(byteStream.begin() + streamIn.pos(), byteStream.end()))); //throw SysError + const std::string& uncompressedStream = decompress(makeStringView(byteStream.begin() + streamIn.pos(), byteStream.end())); //throw SysError + MemoryStreamIn streamInBody(uncompressedStream); auto accessBuf = makeSharedRef<GdriveAccessBuffer>(streamInBody); //throw SysError accessBuf.ref().setContextTimeout(timeoutSec2); //not used by GdriveDrivesBuffer(), but let's be consistent @@ -2874,6 +2882,7 @@ private: try //let's not lose Google Drive data due to unexpected system shutdown: { saveActiveSessions(); } //throw FileError catch (FileError&) { assert(false); } + warn_static("at least log on failure!") }); }; //========================================================================================== @@ -3078,13 +3087,12 @@ void gdriveTraverseFolderRecursive(const GdriveLogin& gdriveLogin, const std::ve struct InputStreamGdrive : public AFS::InputStream { - InputStreamGdrive(const GdrivePath& gdrivePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : - gdrivePath_(gdrivePath), - notifyUnbufferedIO_(notifyUnbufferedIO) + explicit InputStreamGdrive(const GdrivePath& gdrivePath) : + gdrivePath_(gdrivePath) { worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, gdrivePath] { - setCurrentThreadName(Zstr("Istream[Gdrive] ") + utfTo<Zstring>(getGdriveDisplayPath(gdrivePath))); + setCurrentThreadName(Zstr("Istream ") + utfTo<Zstring>(getGdriveDisplayPath(gdrivePath))); try { GdriveAccess access; @@ -3100,12 +3108,11 @@ struct InputStreamGdrive : public AFS::InputStream try { - auto writeBlock = [&](const void* buffer, size_t bytesToWrite) + auto tryWriteBlock = [&](const void* buffer, size_t bytesToWrite) { - return asyncStreamOut->write(buffer, bytesToWrite); //throw ThreadStopRequest + return asyncStreamOut->tryWrite(buffer, bytesToWrite); //throw ThreadStopRequest }; - - gdriveDownloadFile(fileId, writeBlock, access); //throw SysError, ThreadStopRequest + gdriveDownloadFile(fileId, tryWriteBlock, access); //throw SysError, ThreadStopRequest } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getGdriveDisplayPath(gdrivePath))), e.toString()); } @@ -3120,17 +3127,18 @@ struct InputStreamGdrive : public AFS::InputStream asyncStreamIn_->setReadError(std::make_exception_ptr(ThreadStopRequest())); } - size_t read(void* buffer, size_t bytesToRead) override //throw FileError, (ErrorFileLocked), X; return "bytesToRead" bytes unless end of stream! + size_t getBlockSize() override { return GDRIVE_BLOCK_SIZE_DOWNLOAD; } //throw (FileError) + + //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + size_t tryRead(void* buffer, size_t bytesToRead, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, (ErrorFileLocked), X { - const size_t bytesRead = asyncStreamIn_->read(buffer, bytesToRead); //throw FileError - reportBytesProcessed(); //throw X + const size_t bytesRead = asyncStreamIn_->tryRead(buffer, bytesToRead); //throw FileError + reportBytesProcessed(notifyUnbufferedIO); //throw X return bytesRead; //no need for asyncStreamIn_->checkWriteErrors(): once end of stream is reached, asyncStreamOut->closeStream() was called => no errors occured } - size_t getBlockSize() const override { return 64 * 1024; } //non-zero block size is AFS contract! - - std::optional<AFS::StreamAttributes> getAttributesBuffered() override //throw FileError + std::optional<AFS::StreamAttributes> tryGetAttributesFast() override //throw FileError { AFS::StreamAttributes attr = {}; try @@ -3148,15 +3156,14 @@ struct InputStreamGdrive : public AFS::InputStream } private: - void reportBytesProcessed() //throw X + void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { - const int64_t totalBytesDownloaded = asyncStreamIn_->getTotalBytesWritten(); - if (notifyUnbufferedIO_) notifyUnbufferedIO_(totalBytesDownloaded - totalBytesReported_); //throw X - totalBytesReported_ = totalBytesDownloaded; + const int64_t bytesDelta = makeSigned(asyncStreamIn_->getTotalBytesWritten()) - totalBytesReported_; + totalBytesReported_ += bytesDelta; + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } const GdrivePath gdrivePath_; - const IoCallback notifyUnbufferedIO_; //throw X int64_t totalBytesReported_ = 0; std::shared_ptr<AsyncStreamBuffer> asyncStreamIn_ = std::make_shared<AsyncStreamBuffer>(GDRIVE_STREAM_BUFFER_SIZE); InterruptibleThread worker_; @@ -3167,12 +3174,10 @@ private: //already existing: 1. fails or 2. creates duplicate struct OutputStreamGdrive : public AFS::OutputStreamImpl { - OutputStreamGdrive(const GdrivePath& gdrivePath, //throw SysError + OutputStreamGdrive(const GdrivePath& gdrivePath, std::optional<uint64_t> /*streamSize*/, std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/, - std::unique_ptr<PathAccessLock>&& pal) : - notifyUnbufferedIO_(notifyUnbufferedIO) + std::unique_ptr<PathAccessLock>&& pal) //throw SysError { std::promise<AFS::FingerPrint> pFilePrint; futFilePrint_ = pFilePrint.get_future(); @@ -3200,19 +3205,18 @@ struct OutputStreamGdrive : public AFS::OutputStreamImpl pal = std::move(pal)]() mutable { assert(pal); //bind life time to worker thread! - setCurrentThreadName(Zstr("Ostream[Gdrive] ") + utfTo<Zstring>(getGdriveDisplayPath(gdrivePath))); + setCurrentThreadName(Zstr("Ostream ") + utfTo<Zstring>(getGdriveDisplayPath(gdrivePath))); try { - auto readBlock = [&](void* buffer, size_t bytesToRead) + auto tryReadBlock = [&](void* buffer, size_t bytesToRead) //may return short, only 0 means EOF! { - //returns "bytesToRead" bytes unless end of stream! => maps nicely into Posix read() semantics expected by gdriveUploadFile() - return asyncStreamIn->read(buffer, bytesToRead); //throw ThreadStopRequest + return asyncStreamIn->tryRead(buffer, bytesToRead); //throw ThreadStopRequest }; //for whatever reason, gdriveUploadFile() is slightly faster than gdriveUploadSmallFile()! despite its two roundtrips! even when file sizes are 0! //=> 1. issue likely on Google's side => 2. persists even after having fixed "Expect: 100-continue" const std::string fileIdNew = //streamSize && *streamSize < 5 * 1024 * 1024 ? - //gdriveUploadSmallFile(fileName, parentId, *streamSize, modTime, readBlock, aai.access) : //throw SysError, ThreadStopRequest - gdriveUploadFile (fileName, parentId, modTime, readBlock, aai.access); //throw SysError, ThreadStopRequest + //gdriveUploadSmallFile(fileName, parentId, *streamSize, modTime, readBlock, aai.access) : //throw SysError, ThreadStopRequest + gdriveUploadFile (fileName, parentId, modTime, tryReadBlock, aai.access); //throw SysError, ThreadStopRequest assert(asyncStreamIn->getTotalBytesRead() == asyncStreamIn->getTotalBytesWritten()); //already existing: creates duplicate @@ -3251,42 +3255,50 @@ struct OutputStreamGdrive : public AFS::OutputStreamImpl ~OutputStreamGdrive() { - asyncStreamOut_->setWriteError(std::make_exception_ptr(ThreadStopRequest())); + if (asyncStreamOut_) //finalize() was not called (successfully) + asyncStreamOut_->setWriteError(std::make_exception_ptr(ThreadStopRequest())); } - void write(const void* buffer, size_t bytesToWrite) override //throw FileError, X + size_t getBlockSize() override { return GDRIVE_BLOCK_SIZE_UPLOAD; } //throw (FileError) + + size_t tryWrite(const void* buffer, size_t bytesToWrite, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 { - asyncStreamOut_->write(buffer, bytesToWrite); //throw FileError - reportBytesProcessed(); //throw X + const size_t bytesWritten = asyncStreamOut_->tryWrite(buffer, bytesToWrite); //throw FileError + reportBytesProcessed(notifyUnbufferedIO); //throw X + return bytesWritten; } - AFS::FinalizeResult finalize() override //throw FileError, X + AFS::FinalizeResult finalize(const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X { + if (!asyncStreamOut_) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + asyncStreamOut_->closeStream(); while (futFilePrint_.wait_for(std::chrono::milliseconds(50)) == std::future_status::timeout) - reportBytesProcessed(); //throw X - reportBytesProcessed(); //[!] once more, now that *all* bytes were written - - asyncStreamOut_->checkReadErrors(); //throw FileError - //-------------------------------------------------------------------- + reportBytesProcessed(notifyUnbufferedIO); //throw X + reportBytesProcessed(notifyUnbufferedIO); //[!] once more, now that *all* bytes were written AFS::FinalizeResult result; assert(isReady(futFilePrint_)); result.filePrint = futFilePrint_.get(); //throw FileError + + //asyncStreamOut_->checkReadErrors(); //throw FileError -> not needed after *successful* upload + asyncStreamOut_.reset(); //do NOT reset on failure, so that ~OutputStreamGdrive() will request worker thread to stop + //-------------------------------------------------------------------- + //result.errorModTime -> already (successfully) set during file creation return result; } private: - void reportBytesProcessed() //throw X + void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { - const int64_t totalBytesUploaded = asyncStreamOut_->getTotalBytesRead(); - if (notifyUnbufferedIO_) notifyUnbufferedIO_(totalBytesUploaded - totalBytesReported_); //throw X - totalBytesReported_ = totalBytesUploaded; + const int64_t bytesDelta = makeSigned(asyncStreamOut_->getTotalBytesRead()) - totalBytesReported_; + totalBytesReported_ += bytesDelta; + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } - const IoCallback notifyUnbufferedIO_; //throw X int64_t totalBytesReported_ = 0; std::shared_ptr<AsyncStreamBuffer> asyncStreamOut_ = std::make_shared<AsyncStreamBuffer>(GDRIVE_STREAM_BUFFER_SIZE); InterruptibleThread worker_; @@ -3324,11 +3336,11 @@ public: } private: - GdrivePath getGdrivePath(const AfsPath& afsPath) const { return {gdriveLogin_, afsPath}; } + GdrivePath getGdrivePath(const AfsPath& itemPath) const { return {gdriveLogin_, itemPath}; } - GdriveRawPath getGdriveRawPath(const AfsPath& afsPath) const //throw SysError + GdriveRawPath getGdriveRawPath(const AfsPath& itemPath) const //throw SysError { - const std::optional<AfsPath> parentPath = getParentPath(afsPath); + const std::optional<AfsPath> parentPath = getParentPath(itemPath); if (!parentPath) throw SysError(L"Item is device root"); @@ -3337,14 +3349,14 @@ private: { parentId = fileState.getItemId(*parentPath, true /*followLeafShortcut*/); //throw SysError }); - return { std::move(parentId), getItemName(afsPath)}; + return { std::move(parentId), getItemName(itemPath)}; } - Zstring getInitPathPhrase(const AfsPath& afsPath) const override { return concatenateGdriveFolderPathPhrase(getGdrivePath(afsPath)); } + Zstring getInitPathPhrase(const AfsPath& itemPath) const override { return concatenateGdriveFolderPathPhrase(getGdrivePath(itemPath)); } - std::vector<Zstring> getPathPhraseAliases(const AfsPath& afsPath) const override { return {getInitPathPhrase(afsPath)}; } + std::vector<Zstring> getPathPhraseAliases(const AfsPath& itemPath) const override { return {getInitPathPhrase(itemPath)}; } - std::wstring getDisplayPath(const AfsPath& afsPath) const override { return getGdriveDisplayPath(getGdrivePath(afsPath)); } + std::wstring getDisplayPath(const AfsPath& itemPath) const override { return getGdriveDisplayPath(getGdrivePath(itemPath)); } bool isNullFileSystem() const override { return gdriveLogin_.email.empty(); } @@ -3361,21 +3373,21 @@ private: } //---------------------------------------------------------------------------------------------------------------- - ItemType getItemType(const AfsPath& afsPath) const override //throw FileError + ItemType getItemType(const AfsPath& itemPath) const override //throw FileError { - if (const std::optional<ItemType> type = itemStillExists(afsPath)) //throw FileError + if (const std::optional<ItemType> type = itemStillExists(itemPath)) //throw FileError return *type; - throw FileError(replaceCpy(_("Cannot find %x."), L"%x", fmtPath(getDisplayPath(afsPath)))); + throw FileError(replaceCpy(_("Cannot find %x."), L"%x", fmtPath(getDisplayPath(itemPath)))); } - std::optional<ItemType> itemStillExists(const AfsPath& afsPath) const override //throw FileError + std::optional<ItemType> itemStillExists(const AfsPath& itemPath) const override //throw FileError { try { GdriveFileState::PathStatus ps; accessGlobalFileState(gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - ps = fileState.getPathStatus(afsPath, false /*followLeafShortcut*/); //throw SysError + ps = fileState.getPathStatus(itemPath, false /*followLeafShortcut*/); //throw SysError }); if (ps.relPath.empty()) switch (ps.existingType) @@ -3388,23 +3400,23 @@ private: } return {}; } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(itemPath))), e.toString()); } } //---------------------------------------------------------------------------------------------------------------- //already existing: 1. fails or 2. creates duplicate (unlikely) - void createFolderPlain(const AfsPath& afsPath) const override //throw FileError + void createFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { //avoid duplicate Google Drive item creation by multiple threads - PathAccessLock pal(getGdriveRawPath(afsPath), PathBlockType::otherWait); //throw SysError + PathAccessLock pal(getGdriveRawPath(folderPath), PathBlockType::otherWait); //throw SysError - const Zstring folderName = getItemName(afsPath); + const Zstring folderName = getItemName(folderPath); std::string parentId; const GdrivePersistentSessions::AsyncAccessInfo aai = accessGlobalFileState(gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const GdriveFileState::PathStatus& ps = fileState.getPathStatus(afsPath, false /*followLeafShortcut*/); //throw SysError + const GdriveFileState::PathStatus& ps = fileState.getPathStatus(folderPath, false /*followLeafShortcut*/); //throw SysError if (ps.relPath.empty()) throw SysError(replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(folderName))); @@ -3422,20 +3434,20 @@ private: fileState.all().notifyFolderCreated(aai.stateDelta, folderIdNew, folderName, parentId); }); } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeItemPlainImpl(const AfsPath& afsPath, bool permanent /*...or move to trash*/) const //throw SysError + void removeItemPlainImpl(const AfsPath& folderPath, bool permanent /*...or move to trash*/) const //throw SysError { std::string itemId; std::optional<std::string> parentIdToUnlink; const GdrivePersistentSessions::AsyncAccessInfo aai = accessGlobalFileState(gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const std::optional<AfsPath> parentPath = getParentPath(afsPath); + const std::optional<AfsPath> parentPath = getParentPath(folderPath); if (!parentPath) throw SysError(L"Item is device root"); GdriveItemDetails itemDetails; - std::tie(itemId, itemDetails) = fileState.getFileAttributes(afsPath, false /*followLeafShortcut*/); //throw SysError + std::tie(itemId, itemDetails) = fileState.getFileAttributes(folderPath, false /*followLeafShortcut*/); //throw SysError assert(std::find(itemDetails.parentIds.begin(), itemDetails.parentIds.end(), fileState.getItemId(*parentPath, true /*followLeafShortcut*/)) != itemDetails.parentIds.end()); //hard-link handling applies to shared files as well: 1. it's the right thing (TM) 2. if we're not the owner: deleting would fail @@ -3468,59 +3480,59 @@ private: } } - void removeFilePlain(const AfsPath& afsPath) const override //throw FileError + void removeFilePlain(const AfsPath& filePath) const override //throw FileError { - try { removeItemPlainImpl(afsPath, true /*permanent*/); /*throw SysError*/ } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + try { removeItemPlainImpl(filePath, true /*permanent*/); /*throw SysError*/ } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } - void removeSymlinkPlain(const AfsPath& afsPath) const override //throw FileError + void removeSymlinkPlain(const AfsPath& linkPath) const override //throw FileError { - try { removeItemPlainImpl(afsPath, true /*permanent*/); /*throw SysError*/ } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete symbolic link %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + try { removeItemPlainImpl(linkPath, true /*permanent*/); /*throw SysError*/ } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete symbolic link %x."), L"%x", fmtPath(getDisplayPath(linkPath))), e.toString()); } } - void removeFolderPlain(const AfsPath& afsPath) const override //throw FileError + void removeFolderPlain(const AfsPath& folderPath) const override //throw FileError { - try { removeItemPlainImpl(afsPath, true /*permanent*/); /*throw SysError*/ } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + try { removeItemPlainImpl(folderPath, true /*permanent*/); /*throw SysError*/ } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError + void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const override //one call for each object! { - if (onBeforeFolderDeletion) onBeforeFolderDeletion(getDisplayPath(afsPath)); //throw X + if (onBeforeFolderDeletion) onBeforeFolderDeletion(getDisplayPath(folderPath)); //throw X try { //deletes recursively with a single call! - removeFolderPlain(afsPath); //throw FileError + removeFolderPlain(folderPath); //throw FileError } catch (const FileError&) { - if (itemStillExists(afsPath)) //throw FileError + if (itemStillExists(folderPath)) //throw FileError throw; } } //---------------------------------------------------------------------------------------------------------------- - AbstractPath getSymlinkResolvedPath(const AfsPath& afsPath) const override //throw FileError + AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const override //throw FileError { //this function doesn't make sense for Google Drive: Shortcuts do not refer by path, but ID! //even if it were possible to determine a path, doing anything with the target file (e.g. delete + recreate) would break other Shortcuts! - throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(afsPath))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(linkPath))), _("Operation not supported by device.")); } - bool equalSymlinkContentForSameAfsType(const AfsPath& afsLhs, const AbstractPath& apRhs) const override //throw FileError + bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const override //throw FileError { - auto getTargetId = [](const GdriveFileSystem& gdriveFs, const AfsPath& afsPath) + auto getTargetId = [](const GdriveFileSystem& gdriveFs, const AfsPath& linkPath) { try { std::string targetId; const GdrivePersistentSessions::AsyncAccessInfo aai = accessGlobalFileState(gdriveFs.gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const GdriveItemDetails& itemDetails = fileState.getFileAttributes(afsPath, false /*followLeafShortcut*/).second; //throw SysError + const GdriveItemDetails& itemDetails = fileState.getFileAttributes(linkPath, false /*followLeafShortcut*/).second; //throw SysError if (itemDetails.type != GdriveItemType::shortcut) throw SysError(L"Not a Google Drive Shortcut."); @@ -3528,39 +3540,38 @@ private: }); return targetId; } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(gdriveFs.getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(gdriveFs.getDisplayPath(linkPath))), e.toString()); } }; - return getTargetId(*this, afsLhs) == getTargetId(static_cast<const GdriveFileSystem&>(apRhs.afsDevice.ref()), apRhs.afsPath); + return getTargetId(*this, linkPathL) == getTargetId(static_cast<const GdriveFileSystem&>(linkPathR.afsDevice.ref()), linkPathR.afsPath); } //---------------------------------------------------------------------------------------------------------------- //return value always bound: - std::unique_ptr<InputStream> getInputStream(const AfsPath& afsPath, const IoCallback& notifyUnbufferedIO /*throw X*/) const override //throw FileError, (ErrorFileLocked) + std::unique_ptr<InputStream> getInputStream(const AfsPath& filePath) const override //throw FileError, (ErrorFileLocked) { - return std::make_unique<InputStreamGdrive>(getGdrivePath(afsPath), notifyUnbufferedIO); + return std::make_unique<InputStreamGdrive>(getGdrivePath(filePath)); } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: 1. fails or 2. creates duplicate (unlikely) - std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& afsPath, //throw FileError + std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) const override + std::optional<time_t> modTime) const override { try { //avoid duplicate item creation by multiple threads - auto pal = std::make_unique<PathAccessLock>(getGdriveRawPath(afsPath), PathBlockType::otherFail); //throw SysError + auto pal = std::make_unique<PathAccessLock>(getGdriveRawPath(filePath), PathBlockType::otherFail); //throw SysError //don't block during a potentially long-running file upload! //already existing: 1. fails or 2. creates duplicate - return std::make_unique<OutputStreamGdrive>(getGdrivePath(afsPath), streamSize, modTime, notifyUnbufferedIO, std::move(pal)); //throw SysError + return std::make_unique<OutputStreamGdrive>(getGdrivePath(filePath), streamSize, modTime, std::move(pal)); //throw SysError } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } @@ -3574,42 +3585,42 @@ private: //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: 1. fails or 2. creates duplicate (unlikely) - FileCopyResult copyFileForSameAfsType(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), (X) - const AbstractPath& apTarget, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override + FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), (X) + const AbstractPath& targetPath, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override { //no native Google Drive file copy => use stream-based file copy: if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); - const GdriveFileSystem& fsTarget = static_cast<const GdriveFileSystem&>(apTarget.afsDevice.ref()); + const GdriveFileSystem& fsTarget = static_cast<const GdriveFileSystem&>(targetPath.afsDevice.ref()); if (!equalAsciiNoCase(gdriveLogin_.email, fsTarget.gdriveLogin_.email)) //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: 1. fails or 2. creates duplicate (unlikely) - return copyFileAsStream(afsSource, attrSource, apTarget, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X + return copyFileAsStream(sourcePath, attrSource, targetPath, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X //else: copying files within account works, e.g. between My Drive <-> shared drives try { //avoid duplicate Google Drive item creation by multiple threads (blocking is okay: gdriveCopyFile() should complete instantly!) - PathAccessLock pal(fsTarget.getGdriveRawPath(apTarget.afsPath), PathBlockType::otherWait); //throw SysError + PathAccessLock pal(fsTarget.getGdriveRawPath(targetPath.afsPath), PathBlockType::otherWait); //throw SysError - const Zstring itemNameNew = getItemName(apTarget); + const Zstring itemNameNew = getItemName(targetPath); std::string itemIdSrc; GdriveItemDetails itemDetailsSrc; /*const GdrivePersistentSessions::AsyncAccessInfo aaiSrc =*/ accessGlobalFileState(gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - std::tie(itemIdSrc, itemDetailsSrc) = fileState.getFileAttributes(afsSource, true /*followLeafShortcut*/); //throw SysError + std::tie(itemIdSrc, itemDetailsSrc) = fileState.getFileAttributes(sourcePath, true /*followLeafShortcut*/); //throw SysError assert(itemDetailsSrc.type == GdriveItemType::file); //Google Drive *should* fail trying to copy folder: "This file cannot be copied by the user." if (itemDetailsSrc.type != GdriveItemType::file) //=> don't trust + improve error message - throw SysError(replaceCpy<std::wstring>(L"%x is not a file.", L"%x", fmtPath(getItemName(afsSource)))); + throw SysError(replaceCpy<std::wstring>(L"%x is not a file.", L"%x", fmtPath(getItemName(sourcePath)))); }); std::string parentIdTrg; const GdrivePersistentSessions::AsyncAccessInfo aaiTrg = accessGlobalFileState(fsTarget.gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const GdriveFileState::PathStatus psTo = fileState.getPathStatus(apTarget.afsPath, false /*followLeafShortcut*/); //throw SysError + const GdriveFileState::PathStatus psTo = fileState.getPathStatus(targetPath.afsPath, false /*followLeafShortcut*/); //throw SysError if (psTo.relPath.empty()) throw SysError(replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(itemNameNew))); @@ -3640,58 +3651,59 @@ private: fileState.all().notifyItemCreated(aaiTrg.stateDelta, newFileItem); }); - FileCopyResult result; - result.fileSize = itemDetailsSrc.fileSize; - result.modTime = itemDetailsSrc.modTime; - result.sourceFilePrint = getGdriveFilePrint(itemIdSrc); - result.targetFilePrint = getGdriveFilePrint(fileIdTrg); - /*result.errorModTime = */ - return result; + return + { + .fileSize = itemDetailsSrc.fileSize, + .modTime = itemDetailsSrc.modTime, + .sourceFilePrint = getGdriveFilePrint(itemIdSrc), + .targetFilePrint = getGdriveFilePrint(fileIdTrg), + /*.errorModTime = */ + }; } catch (const SysError& e) { throw FileError(replaceCpy(replaceCpy(_("Cannot copy file %x to %y."), - L"%x", L'\n' + fmtPath(getDisplayPath(afsSource))), - L"%y", L'\n' + fmtPath(AFS::getDisplayPath(apTarget))), e.toString()); + L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), + L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))), e.toString()); } } //symlink handling: follow //already existing: fail - void copyNewFolderForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: 1. fails or 2. creates duplicate (unlikely) - AFS::createFolderPlain(apTarget); //throw FileError + AFS::createFolderPlain(targetPath); //throw FileError } //already existing: fail - void copySymlinkForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { try { std::string targetId; accessGlobalFileState(gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const GdriveItemDetails& itemDetails = fileState.getFileAttributes(afsSource, false /*followLeafShortcut*/).second; //throw SysError + const GdriveItemDetails& itemDetails = fileState.getFileAttributes(sourcePath, false /*followLeafShortcut*/).second; //throw SysError if (itemDetails.type != GdriveItemType::shortcut) throw SysError(L"Not a Google Drive Shortcut."); targetId = itemDetails.targetId; }); - const GdriveFileSystem& fsTarget = static_cast<const GdriveFileSystem&>(apTarget.afsDevice.ref()); + const GdriveFileSystem& fsTarget = static_cast<const GdriveFileSystem&>(targetPath.afsDevice.ref()); //avoid duplicate Google Drive item creation by multiple threads - PathAccessLock pal(fsTarget.getGdriveRawPath(apTarget.afsPath), PathBlockType::otherWait); //throw SysError + PathAccessLock pal(fsTarget.getGdriveRawPath(targetPath.afsPath), PathBlockType::otherWait); //throw SysError - const Zstring shortcutName = getItemName(apTarget.afsPath); + const Zstring shortcutName = getItemName(targetPath.afsPath); std::string parentId; const GdrivePersistentSessions::AsyncAccessInfo aaiTrg = accessGlobalFileState(fsTarget.gdriveLogin_, [&](GdriveFileStateAtLocation& fileState) //throw SysError { - const GdriveFileState::PathStatus& ps = fileState.getPathStatus(apTarget.afsPath, false /*followLeafShortcut*/); //throw SysError + const GdriveFileState::PathStatus& ps = fileState.getPathStatus(targetPath.afsPath, false /*followLeafShortcut*/); //throw SysError if (ps.relPath.empty()) throw SysError(replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(shortcutName))); @@ -3712,8 +3724,8 @@ private: catch (const SysError& e) { throw FileError(replaceCpy(replaceCpy(_("Cannot copy symbolic link %x to %y."), - L"%x", L'\n' + fmtPath(getDisplayPath(afsSource))), - L"%y", L'\n' + fmtPath(AFS::getDisplayPath(apTarget))), e.toString()); + L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), + L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))), e.toString()); } } @@ -3787,11 +3799,11 @@ private: catch (const SysError& e) { throw FileError(generateErrorMsg(), e.toString()); } } - bool supportsPermissions(const AfsPath& afsPath) const override { return false; } //throw FileError + bool supportsPermissions(const AfsPath& folderPath) const override { return false; } //throw FileError //---------------------------------------------------------------------------------------------------------------- - FileIconHolder getFileIcon (const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value - ImageHolder getThumbnailImage(const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value + FileIconHolder getFileIcon (const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value + ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value void authenticateAccess(bool allowUserInteraction) const override //throw FileError { @@ -3817,7 +3829,7 @@ private: bool hasNativeTransactionalCopy() const override { return true; } //---------------------------------------------------------------------------------------------------------------- - int64_t getFreeDiskSpace(const AfsPath& afsPath) const override //throw FileError, returns < 0 if not available + int64_t getFreeDiskSpace(const AfsPath& folderPath) const override //throw FileError, returns < 0 if not available { bool onMyDrive = false; try @@ -3830,33 +3842,29 @@ private: else return -1; } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot determine free disk space for %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot determine free disk space for %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - bool supportsRecycleBin(const AfsPath& afsPath) const override { return true; } //throw FileError - - std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& afsPath) const override //throw FileError, return value must be bound! + std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& folderPath) const override //throw FileError, (RecycleBinUnavailable) { struct RecycleSessionGdrive : public RecycleSession { - void recycleItemIfExists(const AbstractPath& itemPath, const Zstring& logicalRelPath) override { AFS::recycleItemIfExists(itemPath); } //throw FileError + //fails if item is not existing + void moveToRecycleBin(const AbstractPath& itemPath, const Zstring& logicalRelPath) override { AFS::moveToRecycleBin(itemPath); } //throw FileError, (RecycleBinUnavailable) void tryCleanup(const std::function<void (const std::wstring& displayPath)>& notifyDeletionStatus) override {}; //throw FileError }; return std::make_unique<RecycleSessionGdrive>(); } - void recycleItemIfExists(const AfsPath& afsPath) const override //throw FileError + //fails if item is not existing + void moveToRecycleBin(const AfsPath& itemPath) const override //throw FileError, (RecycleBinUnavailable) { try { - removeItemPlainImpl(afsPath, false /*permanent*/); //throw SysError - } - catch (const SysError& e) - { - if (itemStillExists(afsPath)) //throw FileError - throw FileError(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + removeItemPlainImpl(itemPath, false /*permanent*/); //throw SysError } + catch (const SysError& e) { throw FileError(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(getDisplayPath(itemPath))), e.toString()); } } const GdriveLogin gdriveLogin_; @@ -4025,7 +4033,7 @@ AbstractPath fff::createItemPathGdrive(const Zstring& itemPathPhrase) //noexcept auto it = std::find_if(fullPath.begin(), fullPath.end(), [](Zchar c) { return c == '/' || c == '\\'; }); const Zstring emailAndDrive(fullPath.begin(), it); - const AfsPath afsPath = sanitizeDeviceRelativePath({it, fullPath.end()}); + const AfsPath itemPath = sanitizeDeviceRelativePath({it, fullPath.end()}); GdriveLogin login; login.email = utfTo<std::string>(beforeFirst(emailAndDrive, Zstr(':'), IfNotFoundReturn::all)); @@ -4037,5 +4045,5 @@ AbstractPath fff::createItemPathGdrive(const Zstring& itemPathPhrase) //noexcept else assert(false); - return AbstractPath(makeSharedRef<GdriveFileSystem>(login), afsPath); + return AbstractPath(makeSharedRef<GdriveFileSystem>(login), itemPath); } diff --git a/FreeFileSync/Source/afs/init_curl_libssh2.cpp b/FreeFileSync/Source/afs/init_curl_libssh2.cpp index 3c8badfe..4d593f3f 100644 --- a/FreeFileSync/Source/afs/init_curl_libssh2.cpp +++ b/FreeFileSync/Source/afs/init_curl_libssh2.cpp @@ -34,6 +34,8 @@ void libsshCurlUnifiedInit() 2019-02-26: following reasons are obsolete due to HAVE_EVP_AES_128_CTR: // - initializes a few statically allocated constants => avoid (minor) race condition if these were initialized by worker threads // - enable proper clean up of these variables in libssh2_exit() (otherwise: memory leaks!) */ + + warn_static("log on error") } diff --git a/FreeFileSync/Source/afs/native.cpp b/FreeFileSync/Source/afs/native.cpp index 8e7a6337..d53aa010 100644 --- a/FreeFileSync/Source/afs/native.cpp +++ b/FreeFileSync/Source/afs/native.cpp @@ -78,19 +78,16 @@ struct NativeFileInfo { FileTimeNative modTime; uint64_t fileSize; - FileIndex fileIndex; + AFS::FingerPrint filePrint; }; -NativeFileInfo getFileAttributes(FileBase::FileHandle fh) //throw SysError +NativeFileInfo getNativeFileInfo(FileBase& file) //throw FileError { - struct stat fileInfo = {}; - if (::fstat(fh, &fileInfo) != 0) - THROW_LAST_SYS_ERROR("fstat"); - + const struct stat& fileInfo = file.getStatBuffered(); //throw FileError return { fileInfo.st_mtim, makeUnsigned(fileInfo.st_size), - fileInfo.st_ino + getFileFingerprint(fileInfo.st_ino) }; } @@ -312,38 +309,41 @@ void traverseFolderRecursiveNative(const std::vector<std::pair<Zstring, std::sha class RecycleSessionNative : public AFS::RecycleSession { public: - explicit RecycleSessionNative(const Zstring& baseFolderPath) : baseFolderPath_(baseFolderPath) {} + explicit RecycleSessionNative(const Zstring& baseFolderPath); //throw FileError, RecycleBinUnavailable - void recycleItemIfExists(const AbstractPath& itemPath, const Zstring& relPath) override; //throw FileError + void moveToRecycleBin(const AbstractPath& itemPath, const Zstring& logicalRelPath) override; //throw FileError, RecycleBinUnavailable void tryCleanup(const std::function<void (const std::wstring& displayPath)>& notifyDeletionStatus /*throw X*/) override; //throw FileError, X private: - const Zstring baseFolderPath_; //ends with path separator }; //=========================================================================================================================== struct InputStreamNative : public AFS::InputStream { - InputStreamNative(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : fi_(filePath, notifyUnbufferedIO) {} //throw FileError, ErrorFileLocked + explicit InputStreamNative(const Zstring& filePath) : fileIn_(filePath) {} //throw FileError, ErrorFileLocked + + size_t getBlockSize() override { return fileIn_.getBlockSize(); } //throw FileError; non-zero block size is AFS contract! - size_t read(void* buffer, size_t bytesToRead) override { return fi_.read(buffer, bytesToRead); } //throw FileError, ErrorFileLocked, X; return "bytesToRead" bytes unless end of stream! - size_t getBlockSize() const override { return fi_.getBlockSize(); } //non-zero block size is AFS contract! - std::optional<AFS::StreamAttributes> getAttributesBuffered() override //throw FileError + //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + size_t tryRead(void* buffer, size_t bytesToRead, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, ErrorFileLocked, X { - try - { - const NativeFileInfo& fileInfo = getFileAttributes(fi_.getHandle()); //throw SysError + const size_t bytesRead = fileIn_.tryRead(buffer, bytesToRead); //throw FileError, ErrorFileLocked + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesRead); //throw X + return bytesRead; + } - return AFS::StreamAttributes({nativeFileTimeToTimeT(fileInfo.modTime), - fileInfo.fileSize, - getFileFingerprint(fileInfo.fileIndex)}); - } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(fi_.getFilePath())), e.toString()); } + std::optional<AFS::StreamAttributes> tryGetAttributesFast() override //throw FileError + { + const NativeFileInfo& fileInfo = getNativeFileInfo(fileIn_); //throw FileError + + return AFS::StreamAttributes({nativeFileTimeToTimeT(fileInfo.modTime), + fileInfo.fileSize, + fileInfo.filePrint}); } private: - FileInput fi_; + FileInputPlain fileIn_; }; //=========================================================================================================================== @@ -352,34 +352,35 @@ struct OutputStreamNative : public AFS::OutputStreamImpl { OutputStreamNative(const Zstring& filePath, std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) : - fo_(filePath, notifyUnbufferedIO), //throw FileError, ErrorTargetExisting + std::optional<time_t> modTime) : + fileOut_(filePath), //throw FileError, ErrorTargetExisting modTime_(modTime) { if (streamSize) //preallocate disk space + reduce fragmentation - fo_.reserveSpace(*streamSize); //throw FileError + fileOut_.reserveSpace(*streamSize); //throw FileError } - void write(const void* buffer, size_t bytesToWrite) override { fo_.write(buffer, bytesToWrite); } //throw FileError, X + size_t getBlockSize() override { return fileOut_.getBlockSize(); } //throw FileError - AFS::FinalizeResult finalize() override //throw FileError, X + size_t tryWrite(const void* buffer, size_t bytesToWrite, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 + { + const size_t bytesWritten = fileOut_.tryWrite(buffer, bytesToWrite); //throw FileError + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesWritten); //throw X + return bytesWritten; + } + + AFS::FinalizeResult finalize(const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X { AFS::FinalizeResult result; if (modTime_) - try - { - const FileIndex fileIndex = getFileAttributes(fo_.getHandle()).fileIndex; //throw SysError - result.filePrint = getFileFingerprint(fileIndex); - } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(fo_.getFilePath())), e.toString()); } + result.filePrint = getNativeFileInfo(fileOut_).filePrint; //throw FileError - fo_.finalize(); //throw FileError, X + fileOut_.close(); //throw FileError try { if (modTime_) - setFileTime(fo_.getFilePath(), *modTime_, ProcSymlink::follow); //throw FileError + setFileTime(fileOut_.getFilePath(), *modTime_, ProcSymlink::follow); //throw FileError /* is setting modtime after closing the file handle a pessimization? no, needed for functional correctness, see file_access.cpp */ } @@ -389,7 +390,7 @@ struct OutputStreamNative : public AFS::OutputStreamImpl } private: - FileOutput fo_; + FileOutputPlain fileOut_; const std::optional<time_t> modTime_; }; @@ -400,20 +401,20 @@ class NativeFileSystem : public AbstractFileSystem public: explicit NativeFileSystem(const Zstring& rootPath) : rootPath_(rootPath) {} - Zstring getNativePath(const AfsPath& afsPath) const { return isNullFileSystem() ? Zstring{} : appendPath(rootPath_, afsPath.value); } + Zstring getNativePath(const AfsPath& itemPath) const { return isNullFileSystem() ? Zstring{} : appendPath(rootPath_, itemPath.value); } private: - Zstring getInitPathPhrase(const AfsPath& afsPath) const override { return makePathPhrase(getNativePath(afsPath)); } + Zstring getInitPathPhrase(const AfsPath& itemPath) const override { return makePathPhrase(getNativePath(itemPath)); } - std::vector<Zstring> getPathPhraseAliases(const AfsPath& afsPath) const override + std::vector<Zstring> getPathPhraseAliases(const AfsPath& itemPath) const override { if (isNullFileSystem()) return {}; - return ::getPathPhraseAliases(getNativePath(afsPath)); + return ::getPathPhraseAliases(getNativePath(itemPath)); } - std::wstring getDisplayPath(const AfsPath& afsPath) const override { return utfTo<std::wstring>(getNativePath(afsPath)); } + std::wstring getDisplayPath(const AfsPath& itemPath) const override { return utfTo<std::wstring>(getNativePath(itemPath)); } bool isNullFileSystem() const override { return rootPath_.empty(); } @@ -423,10 +424,10 @@ private: } //---------------------------------------------------------------------------------------------------------------- - ItemType getItemType(const AfsPath& afsPath) const override //throw FileError + ItemType getItemType(const AfsPath& itemPath) const override //throw FileError { initComForThread(); //throw FileError - switch (zen::getItemType(getNativePath(afsPath))) //throw FileError + switch (zen::getItemType(getNativePath(itemPath))) //throw FileError { case zen::ItemType::file: return AFS::ItemType::file; @@ -439,51 +440,51 @@ private: return AFS::ItemType::file; } - std::optional<ItemType> itemStillExists(const AfsPath& afsPath) const override //throw FileError + std::optional<ItemType> itemStillExists(const AfsPath& itemPath) const override //throw FileError { //default implementation: folder traversal - return AFS::itemStillExists(afsPath); //throw FileError + return AFS::itemStillExists(itemPath); //throw FileError } //---------------------------------------------------------------------------------------------------------------- //already existing: fail - void createFolderPlain(const AfsPath& afsPath) const override //throw FileError + void createFolderPlain(const AfsPath& folderPath) const override //throw FileError { initComForThread(); //throw FileError - createDirectory(getNativePath(afsPath)); //throw FileError, ErrorTargetExisting + createDirectory(getNativePath(folderPath)); //throw FileError, ErrorTargetExisting } - void removeFilePlain(const AfsPath& afsPath) const override //throw FileError + void removeFilePlain(const AfsPath& filePath) const override //throw FileError { initComForThread(); //throw FileError - zen::removeFilePlain(getNativePath(afsPath)); //throw FileError + zen::removeFilePlain(getNativePath(filePath)); //throw FileError } - void removeSymlinkPlain(const AfsPath& afsPath) const override //throw FileError + void removeSymlinkPlain(const AfsPath& linkPath) const override //throw FileError { initComForThread(); //throw FileError - zen::removeSymlinkPlain(getNativePath(afsPath)); //throw FileError + zen::removeSymlinkPlain(getNativePath(linkPath)); //throw FileError } - void removeFolderPlain(const AfsPath& afsPath) const override //throw FileError + void removeFolderPlain(const AfsPath& folderPath) const override //throw FileError { initComForThread(); //throw FileError - zen::removeDirectoryPlain(getNativePath(afsPath)); //throw FileError + zen::removeDirectoryPlain(getNativePath(folderPath)); //throw FileError } - void removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError + void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const override //one call for each object! { //default implementation: folder traversal - AFS::removeFolderIfExistsRecursion(afsPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X + AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X } //---------------------------------------------------------------------------------------------------------------- - AbstractPath getSymlinkResolvedPath(const AfsPath& afsPath) const override //throw FileError + AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const override //throw FileError { initComForThread(); //throw FileError - const Zstring nativePath = getNativePath(afsPath); + const Zstring nativePath = getNativePath(linkPath); const Zstring resolvedPath = zen::getSymlinkResolvedPath(nativePath); //throw FileError const std::optional<zen::PathComponents> comp = parsePathComponents(resolvedPath); @@ -494,14 +495,14 @@ private: return AbstractPath(makeSharedRef<NativeFileSystem>(comp->rootPath), AfsPath(comp->relPath)); } - bool equalSymlinkContentForSameAfsType(const AfsPath& afsLhs, const AbstractPath& apRhs) const override //throw FileError + bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const override //throw FileError { initComForThread(); //throw FileError - const NativeFileSystem& nativeFsR = static_cast<const NativeFileSystem&>(apRhs.afsDevice.ref()); + const NativeFileSystem& nativeFsR = static_cast<const NativeFileSystem&>(linkPathR.afsDevice.ref()); - const SymlinkRawContent linkContentL = getSymlinkRawContent(getNativePath(afsLhs)); //throw FileError - const SymlinkRawContent linkContentR = getSymlinkRawContent(nativeFsR.getNativePath(apRhs.afsPath)); //throw FileError + const SymlinkRawContent linkContentL = getSymlinkRawContent(getNativePath(linkPathL)); //throw FileError + const SymlinkRawContent linkContentR = getSymlinkRawContent(nativeFsR.getNativePath(linkPathR.afsPath)); //throw FileError if (linkContentL.targetPath != linkContentR.targetPath) return false; @@ -511,21 +512,20 @@ private: //---------------------------------------------------------------------------------------------------------------- //return value always bound: - std::unique_ptr<InputStream> getInputStream(const AfsPath& afsPath, const IoCallback& notifyUnbufferedIO /*throw X*/) const override //throw FileError, ErrorFileLocked + std::unique_ptr<InputStream> getInputStream(const AfsPath& filePath) const override //throw FileError, ErrorFileLocked { initComForThread(); //throw FileError - return std::make_unique<InputStreamNative>(getNativePath(afsPath), notifyUnbufferedIO); //throw FileError, ErrorFileLocked + return std::make_unique<InputStreamNative>(getNativePath(filePath)); //throw FileError, ErrorFileLocked } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: fail with clear error message - std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& afsPath, //throw FileError + std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) const override + std::optional<time_t> modTime) const override { initComForThread(); //throw FileError - return std::make_unique<OutputStreamNative>(getNativePath(afsPath), streamSize, modTime, notifyUnbufferedIO); //throw FileError + return std::make_unique<OutputStreamNative>(getNativePath(filePath), streamSize, modTime); //throw FileError, ErrorTargetExisting } //---------------------------------------------------------------------------------------------------------------- @@ -544,21 +544,22 @@ private: //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: fail with clear error message - FileCopyResult copyFileForSameAfsType(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override + FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override { - const Zstring nativePathTarget = static_cast<const NativeFileSystem&>(apTarget.afsDevice.ref()).getNativePath(apTarget.afsPath); + const Zstring nativePathTarget = static_cast<const NativeFileSystem&>(targetPath.afsDevice.ref()).getNativePath(targetPath.afsPath); initComForThread(); //throw FileError - const zen::FileCopyResult nativeResult = copyNewFile(getNativePath(afsSource), nativePathTarget, notifyUnbufferedIO); //throw FileError, ErrorTargetExisting, ErrorFileLocked, X + const zen::FileCopyResult nativeResult = copyNewFile(getNativePath(sourcePath), nativePathTarget, notifyUnbufferedIO); //throw FileError, ErrorTargetExisting, ErrorFileLocked, X //at this point we know we created a new file, so it's fine to delete it for cleanup! ZEN_ON_SCOPE_FAIL(try { zen::removeFilePlain(nativePathTarget); } catch (FileError&) {}); + warn_static("log it!") if (copyFilePermissions) - copyItemPermissions(getNativePath(afsSource), nativePathTarget, ProcSymlink::follow); //throw FileError + copyItemPermissions(getNativePath(sourcePath), nativePathTarget, ProcSymlink::follow); //throw FileError FileCopyResult result; result.fileSize = nativeResult.fileSize; @@ -572,40 +573,40 @@ private: //symlink handling: follow //already existing: fail - void copyNewFolderForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { initComForThread(); //throw FileError - const Zstring& sourcePath = getNativePath(afsSource); - const Zstring& targetPath = static_cast<const NativeFileSystem&>(apTarget.afsDevice.ref()).getNativePath(apTarget.afsPath); + const Zstring& sourcePathNative = getNativePath(sourcePath); + const Zstring& targetPathNative = static_cast<const NativeFileSystem&>(targetPath.afsDevice.ref()).getNativePath(targetPath.afsPath); - zen::createDirectory(targetPath); //throw FileError, ErrorTargetExisting + zen::createDirectory(targetPathNative); //throw FileError, ErrorTargetExisting - ZEN_ON_SCOPE_FAIL(try { removeDirectoryPlain(targetPath); } + ZEN_ON_SCOPE_FAIL(try { removeDirectoryPlain(targetPathNative); } catch (FileError&) {}); + warn_static("log it!") - //do NOT copy attributes for volume root paths which return as: FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY - //https://freefilesync.org/forum/viewtopic.php?t=5550 - if (getParentPath(afsSource)) //=> not a root path - tryCopyDirectoryAttributes(sourcePath, targetPath); //throw FileError + warn_static("implement properly + FileError should lead to a warning only:") + tryCopyDirectoryAttributes(sourcePathNative, targetPathNative); //throw FileError if (copyFilePermissions) - copyItemPermissions(sourcePath, targetPath, ProcSymlink::follow); //throw FileError + copyItemPermissions(sourcePathNative, targetPathNative, ProcSymlink::follow); //throw FileError } //already existing: fail - void copySymlinkForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { - const Zstring nativePathTarget = static_cast<const NativeFileSystem&>(apTarget.afsDevice.ref()).getNativePath(apTarget.afsPath); + const Zstring targetPathNative = static_cast<const NativeFileSystem&>(targetPath.afsDevice.ref()).getNativePath(targetPath.afsPath); initComForThread(); //throw FileError - zen::copySymlink(getNativePath(afsSource), nativePathTarget); //throw FileError + zen::copySymlink(getNativePath(sourcePath), targetPathNative); //throw FileError - ZEN_ON_SCOPE_FAIL(try { zen::removeSymlinkPlain(nativePathTarget); /*throw FileError*/ } + ZEN_ON_SCOPE_FAIL(try { zen::removeSymlinkPlain(targetPathNative); /*throw FileError*/ } catch (FileError&) {}); + warn_static("log it!") if (copyFilePermissions) - copyItemPermissions(getNativePath(afsSource), nativePathTarget, ProcSymlink::asLink); //throw FileError + copyItemPermissions(getNativePath(sourcePath), targetPathNative, ProcSymlink::asLink); //throw FileError } //already existing: undefined behavior! (e.g. fail/overwrite) @@ -624,31 +625,31 @@ private: zen::moveAndRenameItem(getNativePath(pathFrom), nativePathTarget, false /*replaceExisting*/); //throw FileError, ErrorTargetExisting, ErrorMoveUnsupported } - bool supportsPermissions(const AfsPath& afsPath) const override //throw FileError + bool supportsPermissions(const AfsPath& folderPath) const override //throw FileError { initComForThread(); //throw FileError - return zen::supportsPermissions(getNativePath(afsPath)); + return zen::supportsPermissions(getNativePath(folderPath)); } //---------------------------------------------------------------------------------------------------------------- - FileIconHolder getFileIcon(const AfsPath& afsPath, int pixelSize) const override //throw FileError; (optional return value) + FileIconHolder getFileIcon(const AfsPath& filePath, int pixelSize) const override //throw FileError; (optional return value) { initComForThread(); //throw FileError try { - return fff::getFileIcon(getNativePath(afsPath), pixelSize); //throw SysError + return fff::getFileIcon(getNativePath(filePath), pixelSize); //throw SysError } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } - ImageHolder getThumbnailImage(const AfsPath& afsPath, int pixelSize) const override //throw FileError; (optional return value) + ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const override //throw FileError; (optional return value) { initComForThread(); //throw FileError try { - return fff::getThumbnailImage(getNativePath(afsPath), pixelSize); //throw SysError + return fff::getThumbnailImage(getNativePath(filePath), pixelSize); //throw SysError } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } void authenticateAccess(bool allowUserInteraction) const override //throw FileError @@ -660,28 +661,23 @@ private: bool hasNativeTransactionalCopy() const override { return false; } //---------------------------------------------------------------------------------------------------------------- - int64_t getFreeDiskSpace(const AfsPath& afsPath) const override //throw FileError, returns < 0 if not available + int64_t getFreeDiskSpace(const AfsPath& folderPath) const override //throw FileError, returns < 0 if not available { initComForThread(); //throw FileError - return zen::getFreeDiskSpace(getNativePath(afsPath)); //throw FileError - } - - bool supportsRecycleBin(const AfsPath& afsPath) const override //throw FileError - { - return true; //truth be told: no idea!!! + return zen::getFreeDiskSpace(getNativePath(folderPath)); //throw FileError } - std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& afsPath) const override //throw FileError, return value must be bound! + std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& folderPath) const override //throw FileError, RecycleBinUnavailable { initComForThread(); //throw FileError - assert(supportsRecycleBin(afsPath)); - return std::make_unique<RecycleSessionNative>(getNativePath(afsPath)); + return std::make_unique<RecycleSessionNative>(getNativePath(folderPath)); //throw FileError, RecycleBinUnavailable } - void recycleItemIfExists(const AfsPath& afsPath) const override //throw FileError + //fails if item is not existing + void moveToRecycleBin(const AfsPath& itemPath) const override //throw FileError, RecycleBinUnavailable { initComForThread(); //throw FileError - zen::recycleOrDeleteIfExists(getNativePath(afsPath)); //throw FileError + zen::moveToRecycleBin(getNativePath(itemPath)); //throw FileError, RecycleBinUnavailable } const Zstring rootPath_; @@ -689,17 +685,19 @@ private: //=========================================================================================================================== +RecycleSessionNative::RecycleSessionNative(const Zstring& baseFolderPath) +{} -//- return true if item existed +//- fails if item is not existing //- multi-threaded access: internally synchronized! -void RecycleSessionNative::recycleItemIfExists(const AbstractPath& itemPath, const Zstring& relPath) //throw FileError +void RecycleSessionNative::moveToRecycleBin(const AbstractPath& itemPath, const Zstring& logicalRelPath) //throw FileError, RecycleBinUnavailable { const Zstring& itemPathNative = getNativeItemPath(itemPath); if (itemPathNative.empty()) throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - recycleOrDeleteIfExists(itemPathNative); //throw FileError + zen::moveToRecycleBin(itemPathNative); //throw FileError, RecycleBinUnavailable } @@ -744,9 +742,9 @@ AbstractPath fff::createItemPathNativeNoFormatting(const Zstring& nativePath) // } -Zstring fff::getNativeItemPath(const AbstractPath& ap) +Zstring fff::getNativeItemPath(const AbstractPath& itemPath) { - if (const auto nativeDevice = dynamic_cast<const NativeFileSystem*>(&ap.afsDevice.ref())) - return nativeDevice->getNativePath(ap.afsPath); + if (const auto nativeDevice = dynamic_cast<const NativeFileSystem*>(&itemPath.afsDevice.ref())) + return nativeDevice->getNativePath(itemPath.afsPath); return {}; } diff --git a/FreeFileSync/Source/afs/native.h b/FreeFileSync/Source/afs/native.h index 24270e9f..905edd79 100644 --- a/FreeFileSync/Source/afs/native.h +++ b/FreeFileSync/Source/afs/native.h @@ -19,7 +19,7 @@ AbstractPath createItemPathNative(const Zstring& itemPathPhrase); //noexcept AbstractPath createItemPathNativeNoFormatting(const Zstring& nativePath); //noexcept //return empty, if not a native path -Zstring getNativeItemPath(const AbstractPath& ap); +Zstring getNativeItemPath(const AbstractPath& itemPath); } #endif //FS_NATIVE_183247018532434563465 diff --git a/FreeFileSync/Source/afs/sftp.cpp b/FreeFileSync/Source/afs/sftp.cpp index a7544c87..37d7329f 100644 --- a/FreeFileSync/Source/afs/sftp.cpp +++ b/FreeFileSync/Source/afs/sftp.cpp @@ -65,8 +65,8 @@ const long SFTP_DEFAULT_PERMISSION_FOLDER = LIBSSH2_SFTP_S_IRWXU | //attention: if operation fails due to time out, e.g. file copy, the cleanup code may hang, too => total delay = 2 x time out interval -const size_t SFTP_OPTIMAL_BLOCK_SIZE_READ = 8 * MAX_SFTP_READ_SIZE; //https://github.com/libssh2/libssh2/issues/90 -const size_t SFTP_OPTIMAL_BLOCK_SIZE_WRITE = 8 * MAX_SFTP_OUTGOING_SIZE; // +const size_t SFTP_OPTIMAL_BLOCK_SIZE_READ = 16 * MAX_SFTP_READ_SIZE; //https://github.com/libssh2/libssh2/issues/90 +const size_t SFTP_OPTIMAL_BLOCK_SIZE_WRITE = 16 * MAX_SFTP_OUTGOING_SIZE; //need large buffer to mitigate libssh2 stupidly waiting on "acks": https://www.libssh2.org/libssh2_sftp_write.html static_assert(MAX_SFTP_READ_SIZE == 30000 && MAX_SFTP_OUTGOING_SIZE == 30000, "reevaluate optimal block sizes if these constants change!"); /* Perf Test, Sourceforge frs, SFTP upload, compressed 25 MB test file: @@ -84,19 +84,19 @@ SFTP_OPTIMAL_BLOCK_SIZE_READ: SFTP_OPTIMAL_BLOCK_SIZE_WRITE: DSL maximum download speed: 3060 KB/s DSL maximum upload speed: 620 KB/s -Perf Test 2: FFS hompage (2020-04-24) +Perf Test 2: FFS hompage (2022-09-22) SFTP_OPTIMAL_BLOCK_SIZE_READ: SFTP_OPTIMAL_BLOCK_SIZE_WRITE: multiples of multiples of - MAX_SFTP_READ_SIZE MB/s MAX_SFTP_OUTGOING_SIZE KB/s - 1 0,76 1 210 - 2 1,78 2 430 - 4 3,80 4 870 - 8 5,82 8 1178 - 16 5,80 16 1178 - 32 5,80 32 1178 - Filezilla download speed: 5,62 MB/s Filezilla upload speed: 980 KB/s - DSL maximum download speed: 5,96 MB/s DSL maximum upload speed: 1220 KB/s + MAX_SFTP_READ_SIZE MB/s MAX_SFTP_OUTGOING_SIZE MB/s + 1 0,77 1 0.25 + 2 1,63 2 0.50 + 4 3,43 4 0.97 + 8 6,93 8 1.86 + 16 9,41 16 3.60 + 32 9,58 32 3.83 + Filezilla download speed: 12,2 MB/s Filezilla upload speed: 4.4 MB/s -> unfair comparison: FFS seems slower because it includes setup work, e.g. open file handle + DSL maximum download speed: 12,9 MB/s DSL maximum upload speed: 4,7 MB/s => libssh2_sftp_read/libssh2_sftp_write may take quite long for 16x and larger => use smallest multiple that fills bandwidth! */ } @@ -148,16 +148,16 @@ std::weak_ordering operator<=>(const SshSessionId& lhs, const SshSessionId& rhs) namespace { -Zstring concatenateSftpFolderPathPhrase(const SftpLogin& login, const AfsPath& afsPath); //noexcept +Zstring concatenateSftpFolderPathPhrase(const SftpLogin& login, const AfsPath& itemPath); //noexcept -std::string getLibssh2Path(const AfsPath& afsPath) +std::string getLibssh2Path(const AfsPath& itemPath) { - return utfTo<std::string>(getServerRelPath(afsPath)); + return utfTo<std::string>(getServerRelPath(itemPath)); } -std::wstring getSftpDisplayPath(const SshSessionId& sessionId, const AfsPath& afsPath) +std::wstring getSftpDisplayPath(const SshSessionId& sessionId, const AfsPath& itemPath) { Zstring displayPath = Zstring(sftpPrefix) + Zstr("//"); @@ -170,7 +170,7 @@ std::wstring getSftpDisplayPath(const SshSessionId& sessionId, const AfsPath& af port != DEFAULT_PORT_SFTP) displayPath += Zstr(':') + numberTo<Zstring>(port); - const Zstring& relPath = getServerRelPath(afsPath); + const Zstring& relPath = getServerRelPath(itemPath); if (relPath != Zstr("/")) displayPath += relPath; @@ -524,7 +524,7 @@ public: assert(::libssh2_session_last_errno(session->sshSession_) == LIBSSH2_ERROR_EAGAIN); assert(session->nbInfo_.commandPending || std::any_of(session->sftpChannels_.begin(), session->sftpChannels_.end(), [](SftpChannelInfo& ci) { return ci.nbInfo.commandPending; })); - pollfd pfd = {session->socket_->get()}; + pollfd pfd{.fd = session->socket_->get()}; const int dir = ::libssh2_session_block_directions(session->sshSession_); assert(dir != 0); //we assert a blocked direction after libssh2 returned LIBSSH2_ERROR_EAGAIN! @@ -555,7 +555,7 @@ public: //is poll() on macOS broken? https://daniel.haxx.se/blog/2016/10/11/poll-on-mac-10-12-is-broken/ // it seems Daniel only takes issue with "empty" input handling!? => not an issue for us const char* functionName = "poll"; - const int rv = ::poll(&fds[0], //struct pollfd* fds + const int rv = ::poll(fds.data(), //struct pollfd* fds fds.size(), //nfds_t nfds waitTimeMs); //int timeout [ms] if (rv == 0) //time-out! => let next tryNonBlocking() call fail with detailed error! @@ -647,6 +647,7 @@ private: //nbInfo_.commandPending? => have to clean up, no matter what! ::libssh2_session_free(sshSession_); } + warn_static("log on error!") } std::wstring formatLastSshError(const char* functionName, LIBSSH2_SFTP* sftpChannel /*optional*/) const @@ -770,6 +771,7 @@ public: } catch (const SysError& ) { return; } catch (const FatalSshError&) { return; } + warn_static("log on error!") } size_t getSftpChannelCount() const { return session_->getSftpChannelCount(); } @@ -1038,6 +1040,7 @@ std::vector<SftpItem> getDirContentFlat(const SftpLogin& login, const AfsPath& d [&](const SshSession::Details& sd) { return ::libssh2_sftp_closedir(dirHandle); }); //noexcept! } catch (SysError&) {}); + warn_static("log on error!") std::vector<SftpItem> output; for (;;) @@ -1048,40 +1051,40 @@ std::vector<SftpItem> getDirContentFlat(const SftpLogin& login, const AfsPath& d try { runSftpCommand(login, "libssh2_sftp_readdir", //throw SysError - [&](const SshSession::Details& sd) { return rc = ::libssh2_sftp_readdir(dirHandle, &buf[0], buf.size(), &attribs); }); //noexcept! + [&](const SshSession::Details& sd) { return rc = ::libssh2_sftp_readdir(dirHandle, buf.data(), buf.size(), &attribs); }); //noexcept! } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(getSftpDisplayPath(login, dirPath))), e.toString()); } if (rc == 0) //no more items return output; - const std::string_view sftpItemName = makeStringView(&buf[0], rc); + const std::string_view sftpItemName = makeStringView(buf.data(), rc); - if (sftpItemName == "." || sftpItemName == "..") //check needed for SFTP, too! - continue; + if (sftpItemName == "." || sftpItemName == "..") //check needed for SFTP, too! + continue; - const Zstring& itemName = utfTo<Zstring>(sftpItemName); - const AfsPath itemPath(appendPath(dirPath.value, itemName)); + const Zstring& itemName = utfTo<Zstring>(sftpItemName); + const AfsPath itemPath(appendPath(dirPath.value, itemName)); - if ((attribs.flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) == 0) //server probably does not support these attributes => fail at folder level - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"File attributes not available."); + if ((attribs.flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) == 0) //server probably does not support these attributes => fail at folder level + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"File attributes not available."); - if (LIBSSH2_SFTP_S_ISLNK(attribs.permissions)) - { - if ((attribs.flags & LIBSSH2_SFTP_ATTR_ACMODTIME) == 0) //server probably does not support these attributes => fail at folder level - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"Modification time not supported."); - output.push_back({itemName, {AFS::ItemType::symlink, 0, static_cast<time_t>(attribs.mtime)}}); - } - else if (LIBSSH2_SFTP_S_ISDIR(attribs.permissions)) - output.push_back({itemName, {AFS::ItemType::folder, 0, static_cast<time_t>(attribs.mtime)}}); - else //a file or named pipe, ect: LIBSSH2_SFTP_S_ISREG, LIBSSH2_SFTP_S_ISCHR, LIBSSH2_SFTP_S_ISBLK, LIBSSH2_SFTP_S_ISFIFO, LIBSSH2_SFTP_S_ISSOCK - { - if ((attribs.flags & LIBSSH2_SFTP_ATTR_ACMODTIME) == 0) //server probably does not support these attributes => fail at folder level - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"Modification time not supported."); - if ((attribs.flags & LIBSSH2_SFTP_ATTR_SIZE) == 0) - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"File size not supported."); - output.push_back({itemName, {AFS::ItemType::file, attribs.filesize, static_cast<time_t>(attribs.mtime)}}); - } + if (LIBSSH2_SFTP_S_ISLNK(attribs.permissions)) + { + if ((attribs.flags & LIBSSH2_SFTP_ATTR_ACMODTIME) == 0) //server probably does not support these attributes => fail at folder level + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"Modification time not supported."); + output.push_back({itemName, {AFS::ItemType::symlink, 0, static_cast<time_t>(attribs.mtime)}}); + } + else if (LIBSSH2_SFTP_S_ISDIR(attribs.permissions)) + output.push_back({itemName, {AFS::ItemType::folder, 0, static_cast<time_t>(attribs.mtime)}}); + else //a file or named pipe, ect: LIBSSH2_SFTP_S_ISREG, LIBSSH2_SFTP_S_ISCHR, LIBSSH2_SFTP_S_ISBLK, LIBSSH2_SFTP_S_ISFIFO, LIBSSH2_SFTP_S_ISSOCK + { + if ((attribs.flags & LIBSSH2_SFTP_ATTR_ACMODTIME) == 0) //server probably does not support these attributes => fail at folder level + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"Modification time not supported."); + if ((attribs.flags & LIBSSH2_SFTP_ATTR_SIZE) == 0) + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getSftpDisplayPath(login, itemPath))), L"File size not supported."); + output.push_back({itemName, {AFS::ItemType::file, attribs.filesize, static_cast<time_t>(attribs.mtime)}}); + } } } @@ -1202,9 +1205,8 @@ void traverseFolderRecursiveSftp(const SftpLogin& login, const std::vector<std:: struct InputStreamSftp : public AFS::InputStream { - InputStreamSftp(const SftpLogin& login, const AfsPath& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : //throw FileError - displayPath_(getSftpDisplayPath(login, filePath)), - notifyUnbufferedIO_(notifyUnbufferedIO) + InputStreamSftp(const SftpLogin& login, const AfsPath& filePath) : //throw FileError + displayPath_(getSftpDisplayPath(login, filePath)) { try { @@ -1232,53 +1234,18 @@ struct InputStreamSftp : public AFS::InputStream } catch (const SysError&) {} catch (const FatalSshError&) {} //SSH session corrupted! => stop using session + warn_static("log on error?") } - size_t read(void* buffer, size_t bytesToRead) override //throw FileError, (ErrorFileLocked), X; return "bytesToRead" bytes unless end of stream! - { - const size_t blockSize = getBlockSize(); - assert(memBuf_.size() >= blockSize); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); - - auto it = static_cast<std::byte*>(buffer); - const auto itEnd = it + bytesToRead; - for (;;) - { - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), bufPosEnd_ - bufPos_); - std::memcpy(it, &memBuf_[0] + bufPos_, junkSize); - bufPos_ += junkSize; - it += junkSize; - - if (it == itEnd) - break; - //-------------------------------------------------------------------- - const size_t bytesRead = tryRead(&memBuf_[0], blockSize); //throw FileError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 - bufPos_ = 0; - bufPosEnd_ = bytesRead; - - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesRead); //throw X - - if (bytesRead == 0) //end of file - break; - } - return it - static_cast<std::byte*>(buffer); - } - - size_t getBlockSize() const override { return SFTP_OPTIMAL_BLOCK_SIZE_READ; } //non-zero block size is AFS contract! + size_t getBlockSize() override { return SFTP_OPTIMAL_BLOCK_SIZE_READ; } //throw (FileError); non-zero block size is AFS contract! - std::optional<AFS::StreamAttributes> getAttributesBuffered() override //throw FileError - { - return {}; //although have an SFTP stream handle, attribute access requires an extra (expensive) round-trip! - //PERF: test case 148 files, 1MB: overall copy time increases by 20% if libssh2_sftp_fstat() gets called per each file - } - -private: - size_t tryRead(void* buffer, size_t bytesToRead) //throw FileError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + size_t tryRead(void* buffer, size_t bytesToRead, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, (ErrorFileLocked), X { //libssh2_sftp_read has same semantics as Posix read: if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - assert(bytesToRead == getBlockSize()); + assert(bytesToRead % getBlockSize() == 0); ssize_t bytesRead = 0; try @@ -1290,23 +1257,24 @@ private: return static_cast<int>(bytesRead); }); - if (static_cast<size_t>(bytesRead) > bytesToRead) //better safe than sorry + if (makeUnsigned(bytesRead) > bytesToRead) //better safe than sorry throw SysError(formatSystemError("libssh2_sftp_read", L"", L"Buffer overflow.")); //user should never see this } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } catch (const FatalSshError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } //SSH session corrupted! => caller (will/should) stop using session + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesRead); //throw X return bytesRead; //"zero indicates end of file" } + std::optional<AFS::StreamAttributes> tryGetAttributesFast() override { return {}; }//throw FileError + //although we have an SFTP stream handle, attribute access requires an extra (expensive) round-trip! + //PERF: test case 148 files, 1MB: overall copy time increases by 20% if libssh2_sftp_fstat() gets called per each file + +private: const std::wstring displayPath_; LIBSSH2_SFTP_HANDLE* fileHandle_ = nullptr; - const IoCallback notifyUnbufferedIO_; //throw X std::shared_ptr<SftpSessionManager::SshSessionShared> session_; - - std::vector<std::byte> memBuf_ = std::vector<std::byte>(getBlockSize()); - size_t bufPos_ = 0; //buffered I/O; see file_io.cpp - size_t bufPosEnd_ = 0; // }; //=========================================================================================================================== @@ -1316,12 +1284,10 @@ struct OutputStreamSftp : public AFS::OutputStreamImpl { OutputStreamSftp(const SftpLogin& login, //throw FileError const AfsPath& filePath, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) : + std::optional<time_t> modTime) : filePath_(filePath), displayPath_(getSftpDisplayPath(login, filePath)), - modTime_(modTime), - notifyUnbufferedIO_(notifyUnbufferedIO) + modTime_(modTime) { try { @@ -1354,51 +1320,44 @@ struct OutputStreamSftp : public AFS::OutputStreamImpl close(); //throw FileError } catch (FileError&) {} + warn_static("log!?") } - void write(const void* buffer, size_t bytesToWrite) override //throw FileError, X + size_t getBlockSize() override { return SFTP_OPTIMAL_BLOCK_SIZE_WRITE; } //throw (FileError) + + size_t tryWrite(const void* buffer, size_t bytesToWrite, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 { - const size_t blockSize = getBlockSize(); - assert(memBuf_.size() >= blockSize); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); + if (bytesToWrite == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + assert(bytesToWrite % getBlockSize() == 0 || bytesToWrite < getBlockSize()); - auto it = static_cast<const std::byte*>(buffer); - const auto itEnd = it + bytesToWrite; - for (;;) + ssize_t bytesWritten = 0; + try { - if (memBuf_.size() - bufPos_ < blockSize) //support memBuf_.size() > blockSize to reduce memmove()s, but perf test shows: not really needed! - // || bufPos_ == bufPosEnd_) -> not needed while memBuf_.size() == blockSize + session_->executeBlocking("libssh2_sftp_write", //throw SysError, FatalSshError + [&](const SshSession::Details& sd) //noexcept! { - std::memmove(&memBuf_[0], &memBuf_[0] + bufPos_, bufPosEnd_ - bufPos_); - bufPosEnd_ -= bufPos_; - bufPos_ = 0; - } - - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), blockSize - (bufPosEnd_ - bufPos_)); - std::memcpy(&memBuf_[0] + bufPosEnd_, it, junkSize); - bufPosEnd_ += junkSize; - it += junkSize; + bytesWritten = ::libssh2_sftp_write(fileHandle_, static_cast<const char*>(buffer), bytesToWrite); + /* "If this function returns zero it should not be considered an error, but simply that there was no error but yet no payload data got sent to the other end." + => sounds like BS, but is it really true!? + From the libssh2_sftp_write code it appears that the function always waits for at least one "ack", unless we give it so much data _libssh2_channel_write() can't sent it all! */ + assert(bytesWritten != 0); + return static_cast<int>(bytesWritten); + }); - if (it == itEnd) - return; - //-------------------------------------------------------------------- - const size_t bytesWritten = tryWrite(&memBuf_[bufPos_], blockSize); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - bufPos_ += bytesWritten; - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesWritten); //throw X! + if (makeUnsigned(bytesWritten) > bytesToWrite) //better safe than sorry + throw SysError(formatSystemError("libssh2_sftp_write", L"", L"Buffer overflow.")); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } + catch (const FatalSshError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } //SSH session corrupted! => caller (will/should) stop using session + + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesWritten); //throw X! + + return bytesWritten; } - AFS::FinalizeResult finalize() override //throw FileError, X + AFS::FinalizeResult finalize(const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X { - assert(bufPosEnd_ - bufPos_ <= getBlockSize()); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); - while (bufPos_ != bufPosEnd_) - { - const size_t bytesWritten = tryWrite(&memBuf_[bufPos_], bufPosEnd_ - bufPos_); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - bufPos_ += bytesWritten; - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesWritten); //throw X! - } - //~OutputStreamSftp() would call this one, too, but we want to propagate errors if any: close(); //throw FileError @@ -1418,15 +1377,13 @@ struct OutputStreamSftp : public AFS::OutputStreamImpl } private: - size_t getBlockSize() const { return SFTP_OPTIMAL_BLOCK_SIZE_WRITE; } //non-zero block size is AFS contract! - void close() //throw FileError { + if (!fileHandle_) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); try { - if (!fileHandle_) - throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - ZEN_ON_SCOPE_EXIT(fileHandle_ = nullptr); + ZEN_ON_SCOPE_EXIT(fileHandle_ = nullptr); //reset on failure, too! there's no point in, calling libssh2_sftp_close() a second time in ~OutputStreamSftp() session_->executeBlocking("libssh2_sftp_close", //throw SysError, FatalSshError [&](const SshSession::Details& sd) { return ::libssh2_sftp_close(fileHandle_); }); //noexcept! @@ -1435,32 +1392,6 @@ private: catch (const FatalSshError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } //SSH session corrupted! => caller (will/should) stop using session } - size_t tryWrite(const void* buffer, size_t bytesToWrite) //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - { - if (bytesToWrite == 0) - throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - assert(bytesToWrite <= getBlockSize()); - - ssize_t bytesWritten = 0; - try - { - session_->executeBlocking("libssh2_sftp_write", //throw SysError, FatalSshError - [&](const SshSession::Details& sd) //noexcept! - { - bytesWritten = ::libssh2_sftp_write(fileHandle_, static_cast<const char*>(buffer), bytesToWrite); - return static_cast<int>(bytesWritten); - }); - - if (bytesWritten > static_cast<ssize_t>(bytesToWrite)) //better safe than sorry - throw SysError(formatSystemError("libssh2_sftp_write", L"", L"Buffer overflow.")); - } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } - catch (const FatalSshError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayPath_)), e.toString()); } //SSH session corrupted! => caller (will/should) stop using session - - //bytesWritten == 0 is no error according to doc! - return bytesWritten; - } - void setModTimeIfAvailable() const //throw FileError, follows symlinks { assert(!fileHandle_); @@ -1485,12 +1416,7 @@ private: const std::wstring displayPath_; LIBSSH2_SFTP_HANDLE* fileHandle_ = nullptr; const std::optional<time_t> modTime_; - const IoCallback notifyUnbufferedIO_; //throw X std::shared_ptr<SftpSessionManager::SshSessionShared> session_; - - std::vector<std::byte> memBuf_ = std::vector<std::byte>(getBlockSize()); - size_t bufPos_ = 0; //buffered I/O see file_io.cpp - size_t bufPosEnd_ = 0; // }; //=========================================================================================================================== @@ -1514,26 +1440,26 @@ public: } private: - Zstring getInitPathPhrase(const AfsPath& afsPath) const override { return concatenateSftpFolderPathPhrase(login_, afsPath); } + Zstring getInitPathPhrase(const AfsPath& itemPath) const override { return concatenateSftpFolderPathPhrase(login_, itemPath); } - std::vector<Zstring> getPathPhraseAliases(const AfsPath& afsPath) const override + std::vector<Zstring> getPathPhraseAliases(const AfsPath& itemPath) const override { std::vector<Zstring> pathAliases; if (login_.authType != SftpAuthType::keyFile || login_.privateKeyFilePath.empty()) - pathAliases.push_back(concatenateSftpFolderPathPhrase(login_, afsPath)); + pathAliases.push_back(concatenateSftpFolderPathPhrase(login_, itemPath)); else //why going crazy with key path aliases!? because we can... for (const Zstring& pathPhrase : ::getPathPhraseAliases(login_.privateKeyFilePath)) { auto loginTmp = login_; loginTmp.privateKeyFilePath = pathPhrase; - pathAliases.push_back(concatenateSftpFolderPathPhrase(loginTmp, afsPath)); + pathAliases.push_back(concatenateSftpFolderPathPhrase(loginTmp, itemPath)); } return pathAliases; } - std::wstring getDisplayPath(const AfsPath& afsPath) const override { return getSftpDisplayPath(login_, afsPath); } + std::wstring getDisplayPath(const AfsPath& itemPath) const override { return getSftpDisplayPath(login_, itemPath); } bool isNullFileSystem() const override { return login_.server.empty(); } @@ -1557,13 +1483,13 @@ private: } //---------------------------------------------------------------------------------------------------------------- - ItemType getItemType(const AfsPath& afsPath) const override //throw FileError + ItemType getItemType(const AfsPath& itemPath) const override //throw FileError { try { LIBSSH2_SFTP_ATTRIBUTES attr = {}; runSftpCommand(login_, "libssh2_sftp_lstat", //throw SysError - [&](const SshSession::Details& sd) { return ::libssh2_sftp_lstat(sd.sftpChannel, getLibssh2Path(afsPath), &attr); }); //noexcept! + [&](const SshSession::Details& sd) { return ::libssh2_sftp_lstat(sd.sftpChannel, getLibssh2Path(itemPath), &attr); }); //noexcept! if ((attr.flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) == 0) throw SysError(formatSystemError("libssh2_sftp_lstat", L"", L"File attributes not available.")); @@ -1576,19 +1502,19 @@ private: } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(itemPath))), e.toString()); } } - std::optional<ItemType> itemStillExists(const AfsPath& afsPath) const override //throw FileError + std::optional<ItemType> itemStillExists(const AfsPath& itemPath) const override //throw FileError { //default implementation: folder traversal - return AFS::itemStillExists(afsPath); //throw FileError + return AFS::itemStillExists(itemPath); //throw FileError } //---------------------------------------------------------------------------------------------------------------- //already existing: fail - void createFolderPlain(const AfsPath& afsPath) const override //throw FileError + void createFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { @@ -1596,41 +1522,41 @@ private: runSftpCommand(login_, "libssh2_sftp_mkdir", //throw SysError [&](const SshSession::Details& sd) //noexcept! { - return ::libssh2_sftp_mkdir(sd.sftpChannel, getLibssh2Path(afsPath), SFTP_DEFAULT_PERMISSION_FOLDER); - //less explicit variant: return ::libssh2_sftp_mkdir(sd.sftpChannel, getLibssh2Path(afsPath), LIBSSH2_SFTP_DEFAULT_MODE); + return ::libssh2_sftp_mkdir(sd.sftpChannel, getLibssh2Path(folderPath), SFTP_DEFAULT_PERMISSION_FOLDER); + //less explicit variant: return ::libssh2_sftp_mkdir(sd.sftpChannel, getLibssh2Path(folderPath), LIBSSH2_SFTP_DEFAULT_MODE); }); } catch (const SysError& e) //libssh2_sftp_mkdir reports generic LIBSSH2_FX_FAILURE if existing { - throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeFilePlain(const AfsPath& afsPath) const override //throw FileError + void removeFilePlain(const AfsPath& filePath) const override //throw FileError { try { runSftpCommand(login_, "libssh2_sftp_unlink", //throw SysError - [&](const SshSession::Details& sd) { return ::libssh2_sftp_unlink(sd.sftpChannel, getLibssh2Path(afsPath)); }); //noexcept! + [&](const SshSession::Details& sd) { return ::libssh2_sftp_unlink(sd.sftpChannel, getLibssh2Path(filePath)); }); //noexcept! } catch (const SysError& e) { - throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } - void removeSymlinkPlain(const AfsPath& afsPath) const override //throw FileError + void removeSymlinkPlain(const AfsPath& linkPath) const override //throw FileError { - this->removeFilePlain(afsPath); //throw FileError + this->removeFilePlain(linkPath); //throw FileError } - void removeFolderPlain(const AfsPath& afsPath) const override //throw FileError + void removeFolderPlain(const AfsPath& folderPath) const override //throw FileError { int delResult = LIBSSH2_ERROR_NONE; try { runSftpCommand(login_, "libssh2_sftp_rmdir", //throw SysError - [&](const SshSession::Details& sd) { return delResult = ::libssh2_sftp_rmdir(sd.sftpChannel, getLibssh2Path(afsPath)); }); //noexcept! + [&](const SshSession::Details& sd) { return delResult = ::libssh2_sftp_rmdir(sd.sftpChannel, getLibssh2Path(folderPath)); }); //noexcept! } catch (const SysError& e) { @@ -1638,22 +1564,22 @@ private: { //tested: libssh2_sftp_rmdir will fail for symlinks! bool symlinkExists = false; - try { symlinkExists = getItemType(afsPath) == ItemType::symlink; } /*throw FileError*/ catch (FileError&) {} //previous exception is more relevant + try { symlinkExists = getItemType(folderPath) == ItemType::symlink; } /*throw FileError*/ catch (FileError&) {} //previous exception is more relevant if (symlinkExists) - return removeSymlinkPlain(afsPath); //throw FileError + return removeSymlinkPlain(folderPath); //throw FileError } - throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); + throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } - void removeFolderIfExistsRecursion(const AfsPath& afsPath, //throw FileError + void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion) const override //one call for each object! { //default implementation: folder traversal - AFS::removeFolderIfExistsRecursion(afsPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X + AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeFolderDeletion); //throw FileError, X } //---------------------------------------------------------------------------------------------------------------- @@ -1662,61 +1588,64 @@ private: const size_t bufSize = 10000; std::vector<char> buf(bufSize + 1); //ensure buffer is always null-terminated since we don't evaluate the byte count returned by libssh2_sftp_realpath()! + int rc = 0; runSftpCommand(login_, "libssh2_sftp_realpath", //throw SysError - [&](const SshSession::Details& sd) { return ::libssh2_sftp_realpath(sd.sftpChannel, sftpPath, &buf[0], bufSize); }); //noexcept! + [&](const SshSession::Details& sd) { return rc = ::libssh2_sftp_realpath(sd.sftpChannel, sftpPath, buf.data(), bufSize); }); //noexcept! - const std::string sftpPathTrg = &buf[0]; + const std::string_view sftpPathTrg = makeStringView(buf.data(), rc); if (!startsWith(sftpPathTrg, '/')) throw SysError(replaceCpy<std::wstring>(L"Invalid path %x.", L"%x", fmtPath(utfTo<std::wstring>(sftpPathTrg)))); return sanitizeDeviceRelativePath(utfTo<Zstring>(sftpPathTrg)); //code-reuse! but the sanitize part isn't really needed here... } - AbstractPath getSymlinkResolvedPath(const AfsPath& afsPath) const override //throw FileError + AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const override //throw FileError { try { - const AfsPath afsPathTrg = getServerRealPath(getLibssh2Path(afsPath)); //throw SysError - return AbstractPath(makeSharedRef<SftpFileSystem>(login_), afsPathTrg); + const AfsPath linkPathTrg = getServerRealPath(getLibssh2Path(linkPath)); //throw SysError + return AbstractPath(makeSharedRef<SftpFileSystem>(login_), linkPathTrg); } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(linkPath))), e.toString()); } } - bool equalSymlinkContentForSameAfsType(const AfsPath& afsLhs, const AbstractPath& apRhs) const override //throw FileError + bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const override //throw FileError { - auto getTargetPath = [](const SftpFileSystem& sftpFs, const AfsPath& afsPath) + auto getTargetPath = [](const SftpFileSystem& sftpFs, const AfsPath& linkPath) { - const unsigned int bufSize = 10000; - std::string buf(bufSize + 1, '\0'); //ensure buffer is always null-terminated since we don't evaluate the byte count returned by libssh2_sftp_readlink()! + std::string buf(10000, '\0'); + int rc = 0; try { runSftpCommand(sftpFs.login_, "libssh2_sftp_readlink", //throw SysError - [&](const SshSession::Details& sd) { return ::libssh2_sftp_readlink(sd.sftpChannel, getLibssh2Path(afsPath), &buf[0], bufSize); }); //noexcept! + [&](const SshSession::Details& sd) { return rc = ::libssh2_sftp_readlink(sd.sftpChannel, getLibssh2Path(linkPath), buf.data(), buf.size()); }); //noexcept! + + if (makeUnsigned(rc) > buf.size()) //better safe than sorry + throw SysError(formatSystemError("libssh2_sftp_readlink", L"", L"Buffer overflow.")); //user should never see this } - catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(sftpFs.getDisplayPath(afsPath))), e.toString()); } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(sftpFs.getDisplayPath(linkPath))), e.toString()); } - buf.resize(strLength(&buf[0])); + buf.resize(rc); return buf; }; - return getTargetPath(*this, afsLhs) == getTargetPath(static_cast<const SftpFileSystem&>(apRhs.afsDevice.ref()), apRhs.afsPath); + return getTargetPath(*this, linkPathL) == getTargetPath(static_cast<const SftpFileSystem&>(linkPathR.afsDevice.ref()), linkPathR.afsPath); } //---------------------------------------------------------------------------------------------------------------- //return value always bound: - std::unique_ptr<InputStream> getInputStream(const AfsPath& afsPath, const IoCallback& notifyUnbufferedIO /*throw X*/) const override //throw FileError, (ErrorFileLocked) + std::unique_ptr<InputStream> getInputStream(const AfsPath& filePath) const override //throw FileError, (ErrorFileLocked) { - return std::make_unique<InputStreamSftp>(login_, afsPath, notifyUnbufferedIO); //throw FileError + return std::make_unique<InputStreamSftp>(login_, filePath); //throw FileError } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: fail with obscure LIBSSH2_FX_FAILURE error - std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& afsPath, //throw FileError + std::unique_ptr<OutputStreamImpl> getOutputStream(const AfsPath& filePath, //throw FileError std::optional<uint64_t> streamSize, - std::optional<time_t> modTime, - const IoCallback& notifyUnbufferedIO /*throw X*/) const override + std::optional<time_t> modTime) const override { - return std::make_unique<OutputStreamSftp>(login_, afsPath, modTime, notifyUnbufferedIO); //throw FileError + return std::make_unique<OutputStreamSftp>(login_, filePath, modTime); //throw FileError } //---------------------------------------------------------------------------------------------------------------- @@ -1728,34 +1657,34 @@ private: //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - FileCopyResult copyFileForSameAfsType(const AfsPath& afsSource, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), X - const AbstractPath& apTarget, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override + FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), X + const AbstractPath& targetPath, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override { //no native SFTP file copy => use stream-based file copy: if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - return copyFileAsStream(afsSource, attrSource, apTarget, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X + return copyFileAsStream(sourcePath, attrSource, targetPath, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X } //symlink handling: follow //already existing: fail - void copyNewFolderForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override //throw FileError + void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { if (copyFilePermissions) - throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: fail - AFS::createFolderPlain(apTarget); //throw FileError + AFS::createFolderPlain(targetPath); //throw FileError } //already existing: fail - void copySymlinkForSameAfsType(const AfsPath& afsSource, const AbstractPath& apTarget, bool copyFilePermissions) const override + void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override { throw FileError(replaceCpy(replaceCpy(_("Cannot copy symbolic link %x to %y."), - L"%x", L'\n' + fmtPath(getDisplayPath(afsSource))), - L"%y", L'\n' + fmtPath(AFS::getDisplayPath(apTarget))), _("Operation not supported by device.")); + L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), + L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); } //already existing: undefined behavior! (e.g. fail/overwrite) @@ -1795,12 +1724,12 @@ private: } } - bool supportsPermissions(const AfsPath& afsPath) const override { return false; } //throw FileError + bool supportsPermissions(const AfsPath& folderPath) const override { return false; } //throw FileError //wait until there is real demand for copying from and to SFTP with permissions => use stream-based file copy: //---------------------------------------------------------------------------------------------------------------- - FileIconHolder getFileIcon (const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value - ImageHolder getThumbnailImage(const AfsPath& afsPath, int pixelSize) const override { return {}; } //throw FileError; optional return value + FileIconHolder getFileIcon (const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value + ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value void authenticateAccess(bool allowUserInteraction) const override {} //throw FileError @@ -1809,7 +1738,7 @@ private: bool hasNativeTransactionalCopy() const override { return false; } //---------------------------------------------------------------------------------------------------------------- - int64_t getFreeDiskSpace(const AfsPath& afsPath) const override //throw FileError, returns < 0 if not available + int64_t getFreeDiskSpace(const AfsPath& folderPath) const override //throw FileError, returns < 0 if not available { //statvfs is an SFTP v3 extension and not supported by all server implementations //Mikrotik SFTP server fails with LIBSSH2_FX_OP_UNSUPPORTED and corrupts session so that next SFTP call will hang @@ -1835,18 +1764,16 @@ private: #endif } - bool supportsRecycleBin(const AfsPath& afsPath) const override { return false; } //throw FileError - - std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& afsPath) const override //throw FileError, return value must be bound! + std::unique_ptr<RecycleSession> createRecyclerSession(const AfsPath& folderPath) const override //throw FileError, RecycleBinUnavailable { - assert(false); //see supportsRecycleBin() - throw FileError(L"Recycle bin not supported by device."); + throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(folderPath))), + _("Operation not supported by device.")); } - void recycleItemIfExists(const AfsPath& afsPath) const override //throw FileError + void moveToRecycleBin(const AfsPath& itemPath) const override //throw FileError, RecycleBinUnavailable { - assert(false); //see supportsRecycleBin() - throw FileError(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(getDisplayPath(afsPath))), _("Operation not supported by device.")); + throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(itemPath))), + _("Operation not supported by device.")); } const SftpLogin login_; @@ -1855,7 +1782,7 @@ private: //=========================================================================================================================== //expects "clean" login data -Zstring concatenateSftpFolderPathPhrase(const SftpLogin& login, const AfsPath& afsPath) //noexcept +Zstring concatenateSftpFolderPathPhrase(const SftpLogin& login, const AfsPath& folderPath) //noexcept { Zstring username; if (!login.username.empty()) @@ -1865,7 +1792,7 @@ Zstring concatenateSftpFolderPathPhrase(const SftpLogin& login, const AfsPath& a if (login.port > 0) port = Zstr(':') + numberTo<Zstring>(login.port); - Zstring relPath = getServerRelPath(afsPath); + Zstring relPath = getServerRelPath(folderPath); if (relPath == Zstr("/")) relPath.clear(); diff --git a/FreeFileSync/Source/application.cpp b/FreeFileSync/Source/application.cpp index 598eed5c..bc09d4f5 100644 --- a/FreeFileSync/Source/application.cpp +++ b/FreeFileSync/Source/application.cpp @@ -97,7 +97,7 @@ void Application::notifyAppError(const std::wstring& msg, FfsExitCode rc) (msgTypeName.empty() ? L"" : SPACED_DASH + msgTypeName); //error handling strategy unknown and no sync log output available at this point! - std::cerr << '[' + utfTo<std::string>(title) + "] " + utfTo<std::string>(msg) << '\n'; + std::cerr << '[' + utfTo<std::string>(title) + "] " + utfTo<std::string>(msg) + '\n'; //alternative0: std::wcerr: cannot display non-ASCII at all, so why does it exist??? //alternative1: wxSafeShowMessage => NO console output on Debian x86, WTF! //alternative2: wxMessageBox() => works, but we probably shouldn't block during command line usage @@ -122,7 +122,7 @@ bool Application::OnInit() //=> work around 1: bonus: avoid needless DBus calls: https://developer.gnome.org/gio/stable/running-gio-apps.html // drawback: missing MTP and network links in folder picker: https://freefilesync.org/forum/viewtopic.php?t=6871 //if (::setenv("GIO_USE_VFS", "local", true /*overwrite*/) != 0) - // std::cerr << utfTo<std::string>(formatSystemError("setenv(GIO_USE_VFS)", errno)) << "\n"; + // std::cerr << utfTo<std::string>(formatSystemError("setenv(GIO_USE_VFS)", errno)) + '\n'; // //=> work around 2: g_vfs_get_default(); //returns unowned GVfs* @@ -363,7 +363,8 @@ void Application::launch(const std::vector<Zstring>& commandArgs) else if (endsWithAsciiNoCase(filePath, Zstr(".xml"))) globalConfigFile = filePath; else - throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath))); + throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath)), + _("Unexpected file extension:") + L' ' + fmtPath(getFileExtension(filePath))); } } //---------------------------------------------------------------------------------------------------- diff --git a/FreeFileSync/Source/base/algorithm.cpp b/FreeFileSync/Source/base/algorithm.cpp index df760e4e..777be454 100644 --- a/FreeFileSync/Source/base/algorithm.cpp +++ b/FreeFileSync/Source/base/algorithm.cpp @@ -744,7 +744,7 @@ void fff::redetermineSyncDirection(const std::vector<std::pair<BaseFolderPair*, msg += L'\n' + AFS::getDisplayPath(baseFolder->getAbstractPath<SelectSide::left >()) + L' ' + getVariantNameWithSymbol(dirCfg.var) + L' ' + AFS::getDisplayPath(baseFolder->getAbstractPath<SelectSide::right>()); - try { callback.logInfo(msg); /*throw X*/} catch (...) {}; + try { callback.logMessage(msg, PhaseCallback::MsgType::warning); /*throw X*/} catch (...) {}; SetSyncDirectionByConfig::execute(getTwoWayUpdateSet(), *baseFolder); } @@ -1137,11 +1137,9 @@ void copyToAlternateFolderFrom(const std::vector<const FileSystemObject*>& rowsT bool overwriteIfExists, ProcessCallback& callback /*throw X*/) //throw X { - auto notifyItemCopy = [&](const std::wstring& statusText, const std::wstring& displayPath) + auto reportItemInfo = [&](const std::wstring& msgTemplate, const AbstractPath& itemPath) //throw X { - std::wstring msg = replaceCpy(statusText, L"%x", fmtPath(displayPath)); - callback.logInfo(msg); //throw X - callback.updateStatus(std::move(msg)); // + reportInfo(replaceCpy(msgTemplate, L"%x", fmtPath(AFS::getDisplayPath(itemPath))), callback); //throw X }; const std::wstring txtCreatingFile (_("Creating file %x" )); const std::wstring txtCreatingFolder(_("Creating folder %x" )); @@ -1204,7 +1202,7 @@ void copyToAlternateFolderFrom(const std::vector<const FileSystemObject*>& rowsT visitFSObject(*fsObj, [&](const FolderPair& folder) { ItemStatReporter statReporter(1, 0, callback); - notifyItemCopy(txtCreatingFolder, AFS::getDisplayPath(targetPath)); + reportItemInfo(txtCreatingFolder, targetPath); //throw X AFS::createFolderIfMissingRecursion(targetPath); //throw FileError statReporter.reportDelta(1, 0); @@ -1213,9 +1211,11 @@ void copyToAlternateFolderFrom(const std::vector<const FileSystemObject*>& rowsT [&](const FilePair& file) { + ItemStatReporter statReporter(1, file.getFileSize<side>(), callback); + reportItemInfo(txtCreatingFile, targetPath); //throw X + std::wstring statusMsg = replaceCpy(txtCreatingFile, L"%x", fmtPath(AFS::getDisplayPath(targetPath))); - callback.logInfo(statusMsg); //throw X - PercentStatReporter statReporter(std::move(statusMsg), file.getFileSize<side>(), callback); //throw X + PercentStatReporter percentReporter(statusMsg, file.getFileSize<side>(), statReporter); const FileAttributes attr = file.getAttributes<side>(); const AFS::StreamAttributes sourceAttr{attr.modTime, attr.fileSize, attr.filePrint}; @@ -1223,23 +1223,24 @@ void copyToAlternateFolderFrom(const std::vector<const FileSystemObject*>& rowsT copyItem(targetPath, [&](const std::function<void()>& deleteTargetItem) //throw FileError { //already existing + !overwriteIfExists: undefined behavior! (e.g. fail/overwrite/auto-rename) - /*const AFS::FileCopyResult result =*/ AFS::copyFileTransactional(sourcePath, sourceAttr, targetPath, //throw FileError, ErrorFileLocked, X - false /*copyFilePermissions*/, true /*transactionalCopy*/, deleteTargetItem, - [&](int64_t bytesDelta) + const AFS::FileCopyResult result = AFS::copyFileTransactional(sourcePath, sourceAttr, targetPath, //throw FileError, ErrorFileLocked, X + false /*copyFilePermissions*/, true /*transactionalCopy*/, deleteTargetItem, + [&](int64_t bytesDelta) { - statReporter.updateStatus(0, bytesDelta); //throw X - callback.requestUiUpdate(); //throw X => not reliably covered by PercentStatReporter::updateStatus()! e.g. during first few seconds: STATUS_PERCENT_DELAY! + percentReporter.updateDeltaAndStatus(bytesDelta); //throw X + callback.requestUiUpdate(); //throw X => not reliably covered by PercentStatReporter::updateDeltaAndStatus()! e.g. during first few seconds: STATUS_PERCENT_DELAY! }); - //result.errorModTime? => probably irrelevant (behave like Windows Explorer) - warn_static("no, should be logged at least!") + + if (result.errorModTime) //log only; no popup + callback.logMessage(result.errorModTime->toString(), PhaseCallback::MsgType::warning); }); - statReporter.updateStatus(1, 0); //throw X + statReporter.reportDelta(1, 0); }, [&](const SymlinkPair& symlink) { ItemStatReporter statReporter(1, 0, callback); - notifyItemCopy(txtCreatingLink, AFS::getDisplayPath(targetPath)); + reportItemInfo(txtCreatingLink, targetPath); //throw X copyItem(targetPath, [&](const std::function<void()>& deleteTargetItem) //throw FileError { @@ -1248,8 +1249,6 @@ void copyToAlternateFolderFrom(const std::vector<const FileSystemObject*>& rowsT }); statReporter.reportDelta(1, 0); }); - - callback.requestUiUpdate(); //throw X }, callback); //throw X } } @@ -1295,33 +1294,19 @@ namespace { template <SelectSide side> void deleteFromGridAndHDOneSide(std::vector<FileSystemObject*>& rowsToDelete, - bool useRecycleBin, - PhaseCallback& callback) + bool moveToRecycler, + bool& recyclerMissingReportOnce, + bool& warnRecyclerMissing, //WarningDialogs::warnRecyclerMissing + PhaseCallback& callback /*throw X*/) //throw X { - auto notifyItemDeletion = [&](const std::wstring& statusText, const std::wstring& displayPath) - { - std::wstring msg = replaceCpy(statusText, L"%x", fmtPath(displayPath)); - callback.logInfo(msg); //throw X - callback.updateStatus(std::move(msg)); // - }; + const std::wstring txtDelFilePermanent_ = _("Deleting file %x"); + const std::wstring txtDelFileRecycler_ = _("Moving file %x to the recycle bin"); - std::wstring txtRemovingFile; - std::wstring txtRemovingDirectory; - std::wstring txtRemovingSymlink; - - if (useRecycleBin) - { - txtRemovingFile = _("Moving file %x to the recycle bin"); - txtRemovingDirectory = _("Moving folder %x to the recycle bin"); - txtRemovingSymlink = _("Moving symbolic link %x to the recycle bin"); - } - else - { - txtRemovingFile = _("Deleting file %x"); - txtRemovingDirectory = _("Deleting folder %x"); - txtRemovingSymlink = _("Deleting symbolic link %x"); - } + const std::wstring txtDelSymlinkPermanent_ = _("Deleting symbolic link %x"); + const std::wstring txtDelSymlinkRecycler_ = _("Moving symbolic link %x to the recycle bin"); + const std::wstring txtDelFolderPermanent_ = _("Deleting folder %x"); + const std::wstring txtDelFolderRecycler_ = _("Moving folder %x to the recycle bin"); for (FileSystemObject* fsObj : rowsToDelete) //all pointers are required(!) to be bound tryReportingError([&] @@ -1330,105 +1315,111 @@ void deleteFromGridAndHDOneSide(std::vector<FileSystemObject*>& rowsToDelete, if (!fsObj->isEmpty<side>()) //element may be implicitly deleted, e.g. if parent folder was deleted first { - visitFSObject(*fsObj, - [&](const FolderPair& folder) + visitFSObject(*fsObj, [&](const FolderPair& folder) { - if (useRecycleBin) + auto removeFolderPermanently = [&] { - notifyItemDeletion(txtRemovingDirectory, AFS::getDisplayPath(folder.getAbstractPath<side>())); //throw X - - AFS::recycleItemIfExists(folder.getAbstractPath<side>()); //throw FileError - statReporter.reportDelta(1, 0); - } - else - { - auto onBeforeFileDeletion = [&](const std::wstring& displayPath) + auto notifyDeletion = [&](const std::wstring& msgTemplate, const std::wstring& displayPath) { - notifyItemDeletion(txtRemovingFile, displayPath); //throw X - statReporter.reportDelta(1, 0); - }; - auto onBeforeDirDeletion = [&](const std::wstring& displayPath) - { - notifyItemDeletion(txtRemovingDirectory, displayPath); //throw X - statReporter.reportDelta(1, 0); + reportInfo(replaceCpy(msgTemplate, L"%x", fmtPath(displayPath)), statReporter); //throw X + statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done! }; + auto onBeforeFileDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtDelFilePermanent_, displayPath); }; + auto onBeforeDirDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtDelFolderPermanent_, displayPath); }; AFS::removeFolderIfExistsRecursion(folder.getAbstractPath<side>(), onBeforeFileDeletion, onBeforeDirDeletion); //throw FileError + }; + + if (moveToRecycler) + try + { + reportInfo(replaceCpy(txtDelFolderRecycler_, L"%x", fmtPath(AFS::getDisplayPath(folder.getAbstractPath<side>()))), statReporter); //throw X + AFS::moveToRecycleBinIfExists(folder.getAbstractPath<side>()); //throw FileError, RecycleBinUnavailable + statReporter.reportDelta(1, 0); + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce) + { + recyclerMissingReportOnce = true; + callback.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing); //throw X + } + callback.logMessage(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folder.getAbstractPath<side>()))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw X + removeFolderPermanently(); //throw FileError, X + } + else + { + reportInfo(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folder.getAbstractPath<side>()))), statReporter); //throw X + removeFolderPermanently(); //throw FileError, X } }, [&](const FilePair& file) { - notifyItemDeletion(txtRemovingFile, AFS::getDisplayPath(file.getAbstractPath<side>())); //throw X - - if (useRecycleBin) - AFS::recycleItemIfExists(file.getAbstractPath<side>()); //throw FileError + if (moveToRecycler) + try + { + reportInfo(replaceCpy(txtDelFileRecycler_, L"%x", fmtPath(AFS::getDisplayPath(file.getAbstractPath<side>()))), statReporter); //throw X + AFS::moveToRecycleBinIfExists(file.getAbstractPath<side>()); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce) + { + recyclerMissingReportOnce = true; + callback.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing); //throw X + } + callback.logMessage(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(file.getAbstractPath<side>()))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw X + AFS::removeFileIfExists(file.getAbstractPath<side>()); //throw FileError + } else + { + reportInfo(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(file.getAbstractPath<side>()))), statReporter); //throw X AFS::removeFileIfExists(file.getAbstractPath<side>()); //throw FileError + } statReporter.reportDelta(1, 0); }, [&](const SymlinkPair& symlink) { - notifyItemDeletion(txtRemovingSymlink, AFS::getDisplayPath(symlink.getAbstractPath<side>())); //throw X - - if (useRecycleBin) - AFS::recycleItemIfExists(symlink.getAbstractPath<side>()); //throw FileError + if (moveToRecycler) + try + { + reportInfo(replaceCpy(txtDelSymlinkRecycler_, L"%x", fmtPath(AFS::getDisplayPath(symlink.getAbstractPath<side>()))), statReporter); //throw X + AFS::moveToRecycleBinIfExists(symlink.getAbstractPath<side>()); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce) + { + recyclerMissingReportOnce = true; + callback.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing); //throw X + } + callback.logMessage(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(symlink.getAbstractPath<side>()))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw X + AFS::removeSymlinkIfExists(symlink.getAbstractPath<side>()); //throw FileError + } else + { + reportInfo(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(symlink.getAbstractPath<side>()))), statReporter); //throw X AFS::removeSymlinkIfExists(symlink.getAbstractPath<side>()); //throw FileError + } statReporter.reportDelta(1, 0); }); fsObj->removeObject<side>(); //if directory: removes recursively! } - - //remain transactional as much as possible => allow for abort only *after* updating file model - callback.requestUiUpdate(); //throw X }, callback); //throw X } - - -template <SelectSide side> -void categorize(const std::vector<FileSystemObject*>& rows, - std::vector<FileSystemObject*>& deletePermanent, - std::vector<FileSystemObject*>& deleteRecyler, - bool useRecycleBin, - std::map<AbstractPath, bool>& recyclerSupported, - PhaseCallback& callback) //throw X -{ - auto hasRecycler = [&](const AbstractPath& baseFolderPath) -> bool - { - auto it = recyclerSupported.find(baseFolderPath); //perf: avoid duplicate checks! - if (it != recyclerSupported.end()) - return it->second; - - const std::wstring msg = replaceCpy(_("Checking recycle bin availability for folder %x..."), L"%x", fmtPath(AFS::getDisplayPath(baseFolderPath))); - - bool recSupported = false; - tryReportingError([&]{ - recSupported = AFS::supportsRecycleBin(baseFolderPath); //throw FileError - }, callback); //throw X - - recyclerSupported.emplace(baseFolderPath, recSupported); - return recSupported; - }; - - for (FileSystemObject* row : rows) - if (!row->isEmpty<side>()) - { - if (useRecycleBin && hasRecycler(row->base().getAbstractPath<side>())) //Windows' ::SHFileOperation() will delete permanently anyway, but we have a superior deletion routine - deleteRecyler.push_back(row); - else - deletePermanent.push_back(row); - } -} } void fff::deleteFromGridAndHD(const std::vector<FileSystemObject*>& rowsToDeleteOnLeft, //refresh GUI grid after deletion to remove invalid rows const std::vector<FileSystemObject*>& rowsToDeleteOnRight, //all pointers need to be bound! const std::vector<std::pair<BaseFolderPair*, SyncDirectionConfig>>& directCfgs, //attention: rows will be physically deleted! - bool useRecycleBin, + bool moveToRecycler, bool& warnRecyclerMissing, ProcessCallback& callback /*throw X*/) //throw X { @@ -1491,34 +1482,55 @@ void fff::deleteFromGridAndHD(const std::vector<FileSystemObject*>& rowsToDelete }; ZEN_ON_SCOPE_EXIT(updateDirection()); //MSVC: assert is a macro and it doesn't play nice with ZEN_ON_SCOPE_EXIT, surprise... wasn't there something about macros being "evil"? - //categorize rows into permanent deletion and recycle bin - std::vector<FileSystemObject*> deletePermanentLeft; - std::vector<FileSystemObject*> deletePermanentRight; - std::vector<FileSystemObject*> deleteRecylerLeft; - std::vector<FileSystemObject*> deleteRecylerRight; - - std::map<AbstractPath, bool> recyclerSupported; - categorize<SelectSide::left >(deleteLeft, deletePermanentLeft, deleteRecylerLeft, useRecycleBin, recyclerSupported, callback); //throw X - categorize<SelectSide::right>(deleteRight, deletePermanentRight, deleteRecylerRight, useRecycleBin, recyclerSupported, callback); // + bool recyclerMissingReportOnce = false; + deleteFromGridAndHDOneSide<SelectSide::left >(deleteLeft, moveToRecycler, recyclerMissingReportOnce, warnRecyclerMissing, callback); //throw X + deleteFromGridAndHDOneSide<SelectSide::right>(deleteRight, moveToRecycler, recyclerMissingReportOnce, warnRecyclerMissing, callback); // +} - //windows: check if recycle bin really exists; if not, Windows will silently delete, which is wrong - if (useRecycleBin && - std::any_of(recyclerSupported.begin(), recyclerSupported.end(), [](const auto& item) { return !item.second; })) - { - std::wstring msg = _("The recycle bin is not supported by the following folders. Deleted or overwritten files will not be able to be restored:") + L'\n'; +//############################################################################################################ +void fff::deleteListOfFiles(const std::vector<Zstring>& filesToDeletePaths, + std::vector<Zstring>& deletedPaths, + bool moveToRecycler, + bool& warnRecyclerMissing, + ProcessCallback& callback /*throw X*/) //throw X +{ + callback.initNewPhase(filesToDeletePaths.size(), 0 /*bytesTotal*/, ProcessPhase::none); //throw X + assert(deletedPaths.empty()); - for (const auto& [folderPath, supported] : recyclerSupported) - if (!supported) - msg += L'\n' + AFS::getDisplayPath(folderPath); + bool recyclerMissingReportOnce = false; - callback.reportWarning(msg, warnRecyclerMissing); //throw? - } + for (const Zstring& filePath : filesToDeletePaths) + tryReportingError([&] + { + const AbstractPath cfgPath = createItemPathNative(filePath); + ItemStatReporter statReporter(1, 0, callback); - deleteFromGridAndHDOneSide<SelectSide::left>(deleteRecylerLeft, true, callback); - deleteFromGridAndHDOneSide<SelectSide::left>(deletePermanentLeft, false, callback); + if (moveToRecycler) + try + { + reportInfo(replaceCpy(_("Moving file %x to the recycle bin"), L"%x", fmtPath(AFS::getDisplayPath(cfgPath))), callback); //throw X + AFS::moveToRecycleBinIfExists(cfgPath); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce) + { + recyclerMissingReportOnce = true; + callback.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing); //throw X + } + callback.logMessage(replaceCpy(_("Deleting file %x"), L"%x", fmtPath(AFS::getDisplayPath(cfgPath))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw X + AFS::removeFileIfExists(cfgPath); //throw FileError + } + else + { + reportInfo(replaceCpy(_("Deleting file %x"), L"%x", fmtPath(AFS::getDisplayPath(cfgPath))), callback); //throw X + AFS::removeFileIfExists(cfgPath); //throw FileError + } - deleteFromGridAndHDOneSide<SelectSide::right>(deleteRecylerRight, true, callback); - deleteFromGridAndHDOneSide<SelectSide::right>(deletePermanentRight, false, callback); + statReporter.reportDelta(1, 0); + deletedPaths.push_back(filePath); + }, callback); //throw X } //############################################################################################################ @@ -1590,7 +1602,7 @@ void TempFileBuffer::createTempFiles(const std::set<FileDescriptor>& workLoad, P { assert(!tempFilePaths_.contains(descr)); //ensure correct stats, NO overwrite-copy => caller-contract! - MemoryStreamOut<std::string> cookie; //create hash to distinguish different versions and file locations + MemoryStreamOut cookie; //create hash to distinguish different versions and file locations writeNumber (cookie, descr.attr.modTime); writeNumber (cookie, descr.attr.fileSize); writeNumber (cookie, descr.attr.filePrint); @@ -1611,8 +1623,11 @@ void TempFileBuffer::createTempFiles(const std::set<FileDescriptor>& workLoad, P tryReportingError([&] { std::wstring statusMsg = replaceCpy(_("Creating file %x"), L"%x", fmtPath(tempFilePath)); - callback.logInfo(statusMsg); //throw X - PercentStatReporter statReporter(std::move(statusMsg), descr.attr.fileSize, callback); //throw X + + ItemStatReporter statReporter(1, descr.attr.fileSize, callback); + PercentStatReporter percentReporter(statusMsg, descr.attr.fileSize, statReporter); + + reportInfo(std::move(statusMsg), callback); //throw X //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) /*const AFS::FileCopyResult result =*/ @@ -1621,15 +1636,13 @@ void TempFileBuffer::createTempFiles(const std::set<FileDescriptor>& workLoad, P false /*copyFilePermissions*/, true /*transactionalCopy*/, nullptr /*onDeleteTargetFile*/, [&](int64_t bytesDelta) { - statReporter.updateStatus(0, bytesDelta); //throw X - callback.requestUiUpdate(); //throw X => not reliably covered by PercentStatReporter::updateStatus()! e.g. during first few seconds: STATUS_PERCENT_DELAY! + percentReporter.updateDeltaAndStatus(bytesDelta); //throw X + callback.requestUiUpdate(); //throw X => not reliably covered by PercentStatReporter::updateDeltaAndStatus()! e.g. during first few seconds: STATUS_PERCENT_DELAY! }); //result.errorModTime? => irrelevant for temp files! - statReporter.updateStatus(1, 0); //throw X + statReporter.reportDelta(1, 0); tempFilePaths_[descr] = tempFilePath; }, callback); //throw X - - callback.requestUiUpdate(); //throw X } } diff --git a/FreeFileSync/Source/base/algorithm.h b/FreeFileSync/Source/base/algorithm.h index 385d3087..8cc8a87b 100644 --- a/FreeFileSync/Source/base/algorithm.h +++ b/FreeFileSync/Source/base/algorithm.h @@ -64,11 +64,17 @@ void copyToAlternateFolder(std::span<const FileSystemObject* const> rowsToCopyOn void deleteFromGridAndHD(const std::vector<FileSystemObject*>& rowsToDeleteOnLeft, //refresh GUI grid after deletion to remove invalid rows const std::vector<FileSystemObject*>& rowsToDeleteOnRight, //all pointers need to be bound! const std::vector<std::pair<BaseFolderPair*, SyncDirectionConfig>>& directCfgs, //attention: rows will be physically deleted! - bool useRecycleBin, + bool moveToRecycler, //global warnings: bool& warnRecyclerMissing, ProcessCallback& callback /*throw X*/); //throw X +void deleteListOfFiles(const std::vector<Zstring>& filesToDeletePaths, + std::vector<Zstring>& deletedPaths, + bool moveToRecycler, + bool& warnRecyclerMissing, + ProcessCallback& callback /*throw X*/); //throw X + struct FileDescriptor { AbstractPath path; diff --git a/FreeFileSync/Source/base/binary.cpp b/FreeFileSync/Source/base/binary.cpp index 49bdd6eb..a908bfa5 100644 --- a/FreeFileSync/Source/base/binary.cpp +++ b/FreeFileSync/Source/base/binary.cpp @@ -5,139 +5,75 @@ // ***************************************************************************** #include "binary.h" -#include <vector> -#include <chrono> using namespace zen; using namespace fff; using AFS = AbstractFileSystem; -namespace +bool fff::filesHaveSameContent(const AbstractPath& filePath1, const AbstractPath& filePath2, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { -/* -1. there seems to be no perf improvement possible when using file mappings instad of ::ReadFile() calls on Windows: - => buffered access: same perf - => unbuffered access: same perf on USB stick, file mapping 30% slower on local disk + int64_t totalBytesNotified = 0; + IoCallback /*[!] as expected by InputStream::tryRead()*/ notifyIoDiv = IOCallbackDivider(notifyUnbufferedIO, totalBytesNotified); -2. Tests on Win7 x64 show that buffer size does NOT matter if files are located on different physical disks! + const std::unique_ptr<AFS::InputStream> stream1 = AFS::getInputStream(filePath1); //throw FileError + const std::unique_ptr<AFS::InputStream> stream2 = AFS::getInputStream(filePath2); // -Impact of buffer size when files are on same disk: + const size_t blockSize1 = stream1->getBlockSize(); //throw FileError + const size_t blockSize2 = stream2->getBlockSize(); // - buffer MB/s - ------------ - 64 10 - 128 19 - 512 40 - 1024 48 - 2048 56 - 4096 56 - 8192 56 -*/ -const size_t BLOCK_SIZE_MAX = 16 * 1024 * 1024; + const size_t bufCapacity = blockSize2 - 1 + blockSize1 + blockSize2; + const std::unique_ptr<std::byte[]> buf(new std::byte[bufCapacity]); -struct StreamReader -{ - StreamReader(const AbstractPath& filePath, const IoCallback& notifyUnbufferedIO) : //throw FileError - stream_(AFS::getInputStream(filePath, notifyUnbufferedIO)), //throw FileError, ErrorFileLocked - defaultBlockSize_(stream_->getBlockSize()), - dynamicBlockSize_(defaultBlockSize_) { assert(defaultBlockSize_ > 0); } + std::byte* const buf1 = buf.get() + blockSize2; //capacity: blockSize2 - 1 + blockSize1 + std::byte* const buf2 = buf.get(); //capacity: blockSize2 - void appendChunk(std::vector<std::byte>& buffer) //throw FileError, X + size_t buf1PosEnd = 0; + for (;;) { - assert(!eof_); - if (eof_) return; - - buffer.resize(buffer.size() + dynamicBlockSize_); + const size_t bytesRead1 = stream1->tryRead(buf1 + buf1PosEnd, blockSize1, notifyIoDiv); //throw FileError, X; may return short; only 0 means EOF - const auto startTime = std::chrono::steady_clock::now(); - const size_t bytesRead = stream_->read(&*(buffer.end() - dynamicBlockSize_), dynamicBlockSize_); //throw FileError, ErrorFileLocked, X; return "bytesToRead" bytes unless end of stream! - const auto stopTime = std::chrono::steady_clock::now(); - - buffer.resize(buffer.size() - dynamicBlockSize_ + bytesRead); //caveat: unsigned arithmetics - - if (bytesRead < dynamicBlockSize_) + if (bytesRead1 == 0) //end of file { - eof_ = true; - return; - } + size_t buf1Pos = 0; + while (buf1Pos < buf1PosEnd) + { + const size_t bytesRead2 = stream2->tryRead(buf2, blockSize2, notifyIoDiv); //throw FileError, X; may return short; only 0 means EOF - size_t proposedBlockSize = 0; - const auto loopTime = stopTime - startTime; + if (bytesRead2 == 0 ||//end of file + bytesRead2 > buf1PosEnd - buf1Pos) + return false; - if (loopTime >= std::chrono::milliseconds(100)) - lastDelayViolation_ = stopTime; + if (std::memcmp(buf1 + buf1Pos, buf2, bytesRead2) != 0) + return false; - //avoid "flipping back": e.g. DVD-ROMs read 32MB at once, so first read may be > 500 ms, but second one will be 0ms! - if (stopTime >= lastDelayViolation_ + std::chrono::seconds(2)) - { - lastDelayViolation_ = stopTime; - proposedBlockSize = dynamicBlockSize_ * 2; + buf1Pos += bytesRead2; + } + return stream2->tryRead(buf2, blockSize2, notifyIoDiv) == 0; //throw FileError, X; expect EOF } - if (loopTime > std::chrono::milliseconds(500)) - proposedBlockSize = dynamicBlockSize_ / 2; - - if (defaultBlockSize_ <= proposedBlockSize && proposedBlockSize <= BLOCK_SIZE_MAX) - dynamicBlockSize_ = proposedBlockSize; - } - - bool isEof() const { return eof_; } - -private: - const std::unique_ptr<AFS::InputStream> stream_; - const size_t defaultBlockSize_; - size_t dynamicBlockSize_; - std::chrono::steady_clock::time_point lastDelayViolation_ = std::chrono::steady_clock::now(); - bool eof_ = false; -}; -} - - -bool fff::filesHaveSameContent(const AbstractPath& filePath1, const AbstractPath& filePath2, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X -{ - int64_t totalUnbufferedIO = 0; - - StreamReader reader1(filePath1, IOCallbackDivider(notifyUnbufferedIO, totalUnbufferedIO)); //throw FileError - StreamReader reader2(filePath2, IOCallbackDivider(notifyUnbufferedIO, totalUnbufferedIO)); // - - StreamReader* readerLow = &reader1; - StreamReader* readerHigh = &reader2; - - std::vector<std::byte> bufferLow; - std::vector<std::byte> bufferHigh; - - for (;;) - { - readerLow->appendChunk(bufferLow); //throw FileError, X - - if (bufferLow.size() > bufferHigh.size()) + else { - bufferLow.swap(bufferHigh); - std::swap(readerLow, readerHigh); + buf1PosEnd += bytesRead1; + + size_t buf1Pos = 0; + while (buf1PosEnd - buf1Pos >= blockSize2) + { + const size_t bytesRead2 = stream2->tryRead(buf2, blockSize2, notifyIoDiv); //throw FileError, X; may return short; only 0 means EOF + + if (bytesRead2 == 0) //end of file + return false; + + if (std::memcmp(buf1 + buf1Pos, buf2, bytesRead2) != 0) + return false; + + buf1Pos += bytesRead2; + } + if (buf1Pos > 0) + { + buf1PosEnd -= buf1Pos; + std::memmove(buf1, buf1 + buf1Pos, buf1PosEnd); + } } - - if (!std::equal(bufferLow. begin(), bufferLow.end(), - bufferHigh.begin())) - return false; - - if (readerLow->isEof()) - { - if (bufferLow.size() < bufferHigh.size()) - return false; - if (readerHigh->isEof()) - break; - //bufferLow.swap(bufferHigh); not needed - std::swap(readerLow, readerHigh); - } - - //don't let sliding buffer grow too large - bufferHigh.erase(bufferHigh.begin(), bufferHigh.begin() + bufferLow.size()); - bufferLow.clear(); } - - if (totalUnbufferedIO % 2 != 0) - throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - - return true; } diff --git a/FreeFileSync/Source/base/binary.h b/FreeFileSync/Source/base/binary.h index 1223cb05..d635b921 100644 --- a/FreeFileSync/Source/base/binary.h +++ b/FreeFileSync/Source/base/binary.h @@ -12,9 +12,9 @@ namespace fff { -bool filesHaveSameContent(const AbstractPath& filePath1, //throw FileError, X +bool filesHaveSameContent(const AbstractPath& filePath1, const AbstractPath& filePath2, - const zen::IoCallback& notifyUnbufferedIO /*throw X*/); + const zen::IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, X } #endif //BINARY_H_3941281398513241134 diff --git a/FreeFileSync/Source/base/comparison.cpp b/FreeFileSync/Source/base/comparison.cpp index 1a1301bc..220da036 100644 --- a/FreeFileSync/Source/base/comparison.cpp +++ b/FreeFileSync/Source/base/comparison.cpp @@ -129,8 +129,8 @@ ResolvedBaseFolders initializeBaseFolders(const std::vector<FolderPairCfg>& fpCf //--------------------------------------------------------------------------- std::map<std::pair<AfsDevice, ZstringNoCase>, std::set<AbstractPath>> ciPathAliases; - for (const AbstractPath& ap : allFolders) - ciPathAliases[std::pair(ap.afsDevice, ap.afsPath.value)].insert(ap); + for (const AbstractPath& folderPath : allFolders) + ciPathAliases[std::pair(folderPath.afsDevice, folderPath.afsPath.value)].insert(folderPath); if (std::any_of(ciPathAliases.begin(), ciPathAliases.end(), [](const auto& item) { return item.second/*aliases*/.size() > 1; })) { @@ -215,9 +215,10 @@ ComparisonBuffer::ComparisonBuffer(const std::set<DirectoryKey>& folderKeys, const int64_t totalTimeSec = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - compareStartTime).count(); - callback.logInfo(_("Comparison finished:") + L' ' + - _P("1 item found", "%x items found", itemsReported) + SPACED_DASH + - _("Time elapsed:") + L' ' + copyStringTo<std::wstring>(wxTimeSpan::Seconds(totalTimeSec).Format())); //throw X + callback.logMessage(_("Comparison finished:") + L' ' + + _P("1 item found", "%x items found", itemsReported) + SPACED_DASH + + _("Time elapsed:") + L' ' + copyStringTo<std::wstring>(wxTimeSpan::Seconds(totalTimeSec).Format()), + PhaseCallback::MsgType::info); //throw X //------------------------------------------------------------------ //folderStatus_.existing already in buffer, now create entries for the rest: @@ -481,19 +482,23 @@ void categorizeFileByContent(FilePair& file, const std::wstring& txtComparingCon bool haveSameContent = false; const std::wstring errMsg = tryReportingError([&] { - PercentStatReporter statReporter(replaceCpy(txtComparingContentOfFiles, L"%x", fmtPath(file.getRelativePathAny())), - file.getFileSize<SelectSide::left>(), acb); //throw ThreadStopRequest + std::wstring statusMsg = replaceCpy(txtComparingContentOfFiles, L"%x", fmtPath(file.getRelativePathAny())); - //callbacks run *outside* singleThread_ lock! => fine - auto notifyUnbufferedIO = [&statReporter](int64_t bytesDelta) + ItemStatReporter statReporter(1, file.getFileSize<SelectSide::left>(), acb); + PercentStatReporter percentReporter(statusMsg, file.getFileSize<SelectSide::left>(), statReporter); + + acb.updateStatus(std::move(statusMsg)); //throw ThreadStopRequest + + //callbacks run *outside* singleThread lock! => fine + auto notifyUnbufferedIO = [&percentReporter](int64_t bytesDelta) { - statReporter.updateStatus(0, bytesDelta); //throw ThreadStopRequest - interruptionPoint(); //throw ThreadStopRequest => not reliably covered by AsyncPercentStatReporter::updateStatus()! + percentReporter.updateDeltaAndStatus(bytesDelta); //throw ThreadStopRequest + interruptionPoint(); //throw ThreadStopRequest => not reliably covered by PercentStatReporter::updateDeltaAndStatus()! }; haveSameContent = parallel::filesHaveSameContent(file.getAbstractPath<SelectSide::left >(), file.getAbstractPath<SelectSide::right>(), notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest - statReporter.updateStatus(1, 0); //throw ThreadStopRequest + statReporter.reportDelta(1, 0); }, acb); //throw ThreadStopRequest if (!errMsg.empty()) @@ -720,7 +725,7 @@ void forEachSorted(const MapType& fileMap, Function fun) for (const auto& item : fileMap) fileList.push_back(&item); - //sort for natural default sequence on UI file grid: + //sort for natural default sequence on UI file grid: std::sort(fileList.begin(), fileList.end(), [](const FileRef& lhs, const FileRef& rhs) { return compareNoCase(lhs->first /*item name*/, rhs->first) < 0; }); for (const auto& item : fileList) @@ -1027,7 +1032,7 @@ FolderComparison fff::compare(WarningDialogs& warnings, bool createDirLocks, std::unique_ptr<LockHolder>& dirLocks, const std::vector<FolderPairCfg>& fpCfgList, - ProcessCallback& callback) + ProcessCallback& callback /*throw X*/) //throw X { //PERF_START; @@ -1054,7 +1059,7 @@ FolderComparison fff::compare(WarningDialogs& warnings, } catch (const FileError& e) //failure is not critical => log only { - callback.logInfo(e.toString()); //throw X + callback.logMessage(e.toString(), PhaseCallback::MsgType::warning); //throw X } const ResolvedBaseFolders& resInfo = initializeBaseFolders(fpCfgList, @@ -1081,8 +1086,8 @@ FolderComparison fff::compare(WarningDialogs& warnings, haveFullPair = true; if (havePartialPair == haveFullPair) //error if: all empty or exist both full and partial pairs -> support single-folder comparison scenario - callback.reportWarning(_("A folder input field is empty.") + L" \n\n" + //throw X - _("The corresponding folder will be considered as empty."), warnings.warnInputFieldEmpty); + callback.reportWarning(_("A folder input field is empty.") + L" \n\n" + + _("The corresponding folder will be considered as empty."), warnings.warnInputFieldEmpty); //throw X } //Check whether one side is a sub directory of the other side (folder-pair-wise!) diff --git a/FreeFileSync/Source/base/comparison.h b/FreeFileSync/Source/base/comparison.h index 4594c428..b1faf023 100644 --- a/FreeFileSync/Source/base/comparison.h +++ b/FreeFileSync/Source/base/comparison.h @@ -54,7 +54,7 @@ FolderComparison compare(WarningDialogs& warnings, bool createDirLocks, std::unique_ptr<LockHolder>& dirLocks, //out const std::vector<FolderPairCfg>& fpCfgList, - ProcessCallback& callback); + ProcessCallback& callback /*throw X*/); //throw X } #endif //COMPARISON_H_8032178534545426 diff --git a/FreeFileSync/Source/base/db_file.cpp b/FreeFileSync/Source/base/db_file.cpp index a37e336e..3af5a36b 100644 --- a/FreeFileSync/Source/base/db_file.cpp +++ b/FreeFileSync/Source/base/db_file.cpp @@ -66,7 +66,7 @@ AbstractPath getDatabaseFilePath(const BaseFolderPair& baseFolder) void saveStreams(const DbStreams& streamList, const AbstractPath& dbPath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { - MemoryStreamOut<std::string> memStreamOut; + MemoryStreamOut memStreamOut; //write FreeFileSync file identifier writeArray(memStreamOut, DB_FILE_DESCR, sizeof(DB_FILE_DESCR)); @@ -89,12 +89,17 @@ void saveStreams(const DbStreams& streamList, const AbstractPath& dbPath, const //------------------------------------------------------------------------------------------------------------------------ //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - const std::unique_ptr<AFS::OutputStream> fileStreamOut = AFS::getOutputStream(dbPath, //throw FileError + const std::unique_ptr<AFS::OutputStream> fileStreamOut = AFS::getOutputStream(dbPath, memStreamOut.ref().size(), - std::nullopt /*modTime*/, - notifyUnbufferedIO /*throw X*/); - fileStreamOut->write(memStreamOut.ref().c_str(), memStreamOut.ref().size()); //throw FileError, X - fileStreamOut->finalize(); //throw FileError, X + std::nullopt /*modTime*/); //throw FileError + + unbufferedSave(memStreamOut.ref(), [&](const void* buffer, size_t bytesToWrite) + { + return fileStreamOut->tryWrite(buffer, bytesToWrite, notifyUnbufferedIO); //throw FileError, X + }, + fileStreamOut->getBlockSize()); //throw FileError, X + + fileStreamOut->finalize(notifyUnbufferedIO); //throw FileError, X } @@ -104,8 +109,13 @@ DbStreams loadStreams(const AbstractPath& dbPath, const IoCallback& notifyUnbuff std::string byteStream; try { - const std::unique_ptr<AFS::InputStream> fileStreamIn = AFS::getInputStream(dbPath, notifyUnbufferedIO); //throw FileError, ErrorFileLocked - byteStream = bufferedLoad<std::string>(*fileStreamIn); //throw FileError, ErrorFileLocked, X + const std::unique_ptr<AFS::InputStream> fileIn = AFS::getInputStream(dbPath); //throw FileError, ErrorFileLocked + + byteStream = unbufferedLoad<std::string>([&](void* buffer, size_t bytesToRead) + { + return fileIn->tryRead(buffer, bytesToRead, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X; may return short, only 0 means EOF! + }, + fileIn->getBlockSize()); //throw FileError, X } catch (const FileError& e) { @@ -139,7 +149,7 @@ DbStreams loadStreams(const AbstractPath& dbPath, const IoCallback& notifyUnbuff // => only "partially" useful for container/stream metadata since the streams data is zlib-compressed { assert(byteStream.size() >= sizeof(uint32_t)); //obviously in this context! - MemoryStreamOut<std::string> crcStreamOut; + MemoryStreamOut crcStreamOut; writeNumber<uint32_t>(crcStreamOut, getCrc32(byteStream.begin(), byteStream.end() - sizeof(uint32_t))); if (!endsWith(byteStream, crcStreamOut.ref())) @@ -195,8 +205,8 @@ public: std::string& streamL, std::string& streamR) { - MemoryStreamOut<std::string> outL; - MemoryStreamOut<std::string> outR; + MemoryStreamOut outL; + MemoryStreamOut outR; //save format version writeNumber<int32_t>(outL, DB_STREAM_VERSION); writeNumber<int32_t>(outR, DB_STREAM_VERSION); @@ -234,7 +244,7 @@ public: const std::string bufSmallNum = compStream(generator.streamOutSmallNum_.ref()); const std::string bufBigNum = compStream(generator.streamOutBigNum_ .ref()); - MemoryStreamOut<std::string> streamOut; + MemoryStreamOut streamOut; writeContainer(streamOut, bufText); writeContainer(streamOut, bufSmallNum); writeContainer(streamOut, bufBigNum); @@ -248,8 +258,8 @@ public: writeNumber<uint64_t>(outL, size1stPart); writeNumber<uint64_t>(outR, size2ndPart); - if (size1stPart > 0) writeArray(outL, &buf[0], size1stPart); - if (size2ndPart > 0) writeArray(outR, &buf[0] + size1stPart, size2ndPart); + if (size1stPart > 0) writeArray(outL, buf.c_str(), size1stPart); + if (size2ndPart > 0) writeArray(outR, buf.c_str() + size1stPart, size2ndPart); streamL = std::move(outL.ref()); streamR = std::move(outR.ref()); @@ -307,9 +317,9 @@ private: - use null-termination in writeItemName() => 5% size reduction (embedded zeros impossible?) - use empty item name as sentinel => only 0,17% size reduction! - save fileSize using instreamOutBigNum_ => pessimization! */ - MemoryStreamOut<std::string> streamOutText_; // - MemoryStreamOut<std::string> streamOutSmallNum_; //data with bias to lead side (= always left in this context) - MemoryStreamOut<std::string> streamOutBigNum_; // + MemoryStreamOut streamOutText_; // + MemoryStreamOut streamOutSmallNum_; //data with bias to lead side (= always left in this context) + MemoryStreamOut streamOutBigNum_; // }; @@ -344,15 +354,15 @@ public: if (has1stPartL != leadStreamLeft) throw SysError(_("File content is corrupted.") + L" (has1stPartL != leadStreamLeft)"); - MemoryStreamIn<std::string>& in1stPart = leadStreamLeft ? streamInL : streamInR; - MemoryStreamIn<std::string>& in2ndPart = leadStreamLeft ? streamInR : streamInL; + MemoryStreamIn& in1stPart = leadStreamLeft ? streamInL : streamInR; + MemoryStreamIn& in2ndPart = leadStreamLeft ? streamInR : streamInL; const size_t size1stPart = static_cast<size_t>(readNumber<uint64_t>(in1stPart)); const size_t size2ndPart = static_cast<size_t>(readNumber<uint64_t>(in2ndPart)); std::string tmpB(size1stPart + size2ndPart, '\0'); //throw std::bad_alloc - readArray(in1stPart, &tmpB[0], size1stPart); //stream always non-empty - readArray(in2ndPart, &tmpB[0] + size1stPart, size2ndPart); //throw SysErrorUnexpectedEos + readArray(in1stPart, tmpB.data(), size1stPart); //stream always non-empty + readArray(in2ndPart, tmpB.data() + size1stPart, size2ndPart); //throw SysErrorUnexpectedEos const std::string tmpL = readContainer<std::string>(streamInL); const std::string tmpR = readContainer<std::string>(streamInR); @@ -367,15 +377,15 @@ public: else if (streamVersion == 3 || //TODO: remove migration code at some time! 2021-02-14 streamVersion == DB_STREAM_VERSION) { - MemoryStreamIn<std::string>& streamInPart1 = leadStreamLeft ? streamInL : streamInR; - MemoryStreamIn<std::string>& streamInPart2 = leadStreamLeft ? streamInR : streamInL; + MemoryStreamIn& streamInPart1 = leadStreamLeft ? streamInL : streamInR; + MemoryStreamIn& streamInPart2 = leadStreamLeft ? streamInR : streamInL; const size_t sizePart1 = static_cast<size_t>(readNumber<uint64_t>(streamInPart1)); const size_t sizePart2 = static_cast<size_t>(readNumber<uint64_t>(streamInPart2)); std::string buf(sizePart1 + sizePart2, '\0'); - if (sizePart1 > 0) readArray(streamInPart1, &buf[0], sizePart1); //throw SysErrorUnexpectedEos - if (sizePart2 > 0) readArray(streamInPart2, &buf[0] + sizePart1, sizePart2); // + if (sizePart1 > 0) readArray(streamInPart1, buf.data(), sizePart1); //throw SysErrorUnexpectedEos + if (sizePart2 > 0) readArray(streamInPart2, buf.data() + sizePart1, sizePart2); // MemoryStreamIn streamIn(buf); const std::string bufText = readContainer<std::string>(streamIn); // @@ -403,11 +413,14 @@ public: } private: - StreamParser(int streamVersion, const std::string& bufText, const std::string& bufSmallNumbers, const std::string& bufBigNumbers) : + StreamParser(int streamVersion, + std::string&& bufText, + std::string&& bufSmallNumbers, + std::string&& bufBigNumbers) : streamVersion_(streamVersion), - streamInText_(bufText), - streamInSmallNum_(bufSmallNumbers), - streamInBigNum_(bufBigNumbers) {} + bufText_ (std::move(bufText)), + bufSmallNumbers_(std::move(bufSmallNumbers)), + bufBigNumbers_ (std::move(bufBigNumbers)) {} template <SelectSide leadSide> void recurse(InSyncFolder& container) //throw SysError @@ -480,12 +493,12 @@ private: class StreamParserV2 { public: - StreamParserV2(const std::string& bufferL, - const std::string& bufferR, - const std::string& bufferB) : - inputLeft_ (bufferL), - inputRight_(bufferR), - inputBoth_ (bufferB) {} + StreamParserV2(std::string&& bufferL, + std::string&& bufferR, + std::string&& bufferB) : + bufL_(std::move(bufferL)), + bufR_(std::move(bufferR)), + bufB_(std::move(bufferB)) {} void recurse(InSyncFolder& container) //throw SysError { @@ -524,15 +537,21 @@ private: } private: - MemoryStreamIn<std::string> inputLeft_; //data related to one side only - MemoryStreamIn<std::string> inputRight_; // - MemoryStreamIn<std::string> inputBoth_; //data concerning both sides + const std::string bufL_; + const std::string bufR_; + const std::string bufB_; + MemoryStreamIn inputLeft_ {bufL_}; //data related to one side only + MemoryStreamIn inputRight_{bufR_}; // + MemoryStreamIn inputBoth_ {bufB_}; //data concerning both sides }; const int streamVersion_; - MemoryStreamIn<std::string> streamInText_; // - MemoryStreamIn<std::string> streamInSmallNum_; //data with bias to lead side - MemoryStreamIn<std::string> streamInBigNum_; // + const std::string bufText_; + const std::string bufSmallNumbers_; + const std::string bufBigNumbers_ ; + MemoryStreamIn streamInText_ {bufText_}; // + MemoryStreamIn streamInSmallNum_{bufSmallNumbers_}; //data with bias to lead side + MemoryStreamIn streamInBigNum_ {bufBigNumbers_}; // }; //####################################################################################################################################### @@ -713,7 +732,7 @@ private: struct StreamStatusNotifier { StreamStatusNotifier(const std::wstring& statusMsg, AsyncCallback& acb /*throw ThreadStopRequest*/) : - msgPrefix_(statusMsg), acb_(acb) {} + msgPrefix_(statusMsg + L' '), acb_(acb) {} void operator()(int64_t bytesDelta) //throw ThreadStopRequest { @@ -973,6 +992,7 @@ void fff::saveLastSynchronousState(const BaseFolderPair& baseFolder, bool transa saveStreams(streams, dbPathTmp, notifySave); //throw FileError, ThreadStopRequest ZEN_ON_SCOPE_FAIL(try { AFS::removeFilePlain(dbPathTmp); } catch (FileError&) {}); + warn_static("log it!") //operation finished: rename temp file -> this should work (almost) transactionally: //if there were no write access, creation of temp file would have failed diff --git a/FreeFileSync/Source/base/dir_lock.cpp b/FreeFileSync/Source/base/dir_lock.cpp index ee49cb9d..58328365 100644 --- a/FreeFileSync/Source/base/dir_lock.cpp +++ b/FreeFileSync/Source/base/dir_lock.cpp @@ -126,7 +126,7 @@ private: catch (const SysError& e) { const std::wstring logMsg = replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(lockFilePath_)) + L' ' + e.toString(); - std::cerr << utfTo<std::string>(logMsg) << '\n'; + std::cerr << utfTo<std::string>(logMsg) + '\n'; } } @@ -177,13 +177,13 @@ LockInformation getLockInfoFromCurrentProcess() //throw FileError //wxGetFullHostName() is a performance killer and can hang for some users, so don't touch! std::vector<char> buf(10000); - if (::gethostname(&buf[0], buf.size()) != 0) + if (::gethostname(buf.data(), buf.size()) != 0) THROW_LAST_FILE_ERROR(_("Cannot get process information."), "gethostname"); - lockInfo.computerName = osName + ' ' + &buf[0] + '.'; + lockInfo.computerName = osName + ' ' + buf.data() + '.'; - if (::getdomainname(&buf[0], buf.size()) != 0) + if (::getdomainname(buf.data(), buf.size()) != 0) THROW_LAST_FILE_ERROR(_("Cannot get process information."), "getdomainname"); - lockInfo.computerName += &buf[0]; //can be "(none)"! + lockInfo.computerName += buf.data(); //can be "(none)"! lockInfo.processId = ::getpid(); //never fails @@ -198,7 +198,7 @@ LockInformation getLockInfoFromCurrentProcess() //throw FileError std::string serialize(const LockInformation& lockInfo) { - MemoryStreamOut<std::string> streamOut; + MemoryStreamOut streamOut; writeArray(streamOut, LOCK_FILE_DESCR, sizeof(LOCK_FILE_DESCR)); writeNumber<int32_t>(streamOut, LOCK_FILE_VERSION); @@ -239,7 +239,7 @@ LockInformation unserialize(const std::string& byteStream) //throw SysError const std::string_view byteStreamTrm = makeStringView(byteStream.begin(), posEnd); - MemoryStreamOut<std::string> crcStreamOut; + MemoryStreamOut crcStreamOut; writeNumber<uint32_t>(crcStreamOut, getCrc32(byteStreamTrm.begin(), byteStreamTrm.end() - sizeof(uint32_t))); if (!endsWith(byteStreamTrm, crcStreamOut.ref())) @@ -418,6 +418,7 @@ void releaseLock(const Zstring& lockFilePath) //noexcept removeFilePlain(lockFilePath); //throw FileError } catch (FileError&) {} + warn_static("log!!! at the very least") } @@ -441,13 +442,18 @@ bool tryLock(const Zstring& lockFilePath) //throw FileError THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(lockFilePath)), "open"); } - FileOutput fileOut(hFile, lockFilePath, nullptr /*notifyUnbufferedIO*/); //pass handle ownership + FileOutputPlain fileOut(hFile, lockFilePath); //pass handle ownership //write housekeeping info: user, process info, lock GUID const std::string byteStream = serialize(getLockInfoFromCurrentProcess()); //throw FileError - fileOut.write(byteStream.c_str(), byteStream.size()); //throw FileError, (X) - fileOut.finalize(); // + unbufferedSave(byteStream, [&](const void* buffer, size_t bytesToWrite) + { + return fileOut.tryWrite(buffer, bytesToWrite); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 + }, + fileOut.getBlockSize()); + + fileOut.close(); //throw FileError return true; } } diff --git a/FreeFileSync/Source/base/file_hierarchy.h b/FreeFileSync/Source/base/file_hierarchy.h index 2062a4ce..9e772da6 100644 --- a/FreeFileSync/Source/base/file_hierarchy.h +++ b/FreeFileSync/Source/base/file_hierarchy.h @@ -837,8 +837,7 @@ bool FileSystemObject::isPairEmpty() const template <SelectSide side> inline Zstring FileSystemObject::getItemName() const { - assert(!itemNameL_.empty() || !itemNameR_.empty()); //-> file pair might be empty (until removed after sync) - //=> okay, but where does this trigger!? calling this function in this case is a bug! + //assert(!itemNameL_.empty() || !itemNameR_.empty()); //-> file pair might be temporarily empty (until permanently removed after sync) const Zstring& itemName = selectParam<side>(itemNameL_, itemNameR_); //empty if not existing if (!itemName.empty()) //avoid ternary-WTF! (implicit copy-constructor call!!!!!!) diff --git a/FreeFileSync/Source/base/parallel_scan.cpp b/FreeFileSync/Source/base/parallel_scan.cpp index 2352fec4..389c67bb 100644 --- a/FreeFileSync/Source/base/parallel_scan.cpp +++ b/FreeFileSync/Source/base/parallel_scan.cpp @@ -138,8 +138,8 @@ public: { std::lock_guard dummy(lockCurrentStatus_); - [[maybe_unused]] const auto it = activeThreadIdxs_.emplace(threadIdx, parallelOps); - assert(it.second); + [[maybe_unused]] const auto [it, inserted] = activeThreadIdxs_.emplace(threadIdx, parallelOps); + assert(inserted); notifyingThreadIdx_ = activeThreadIdxs_.begin()->first; } @@ -223,11 +223,11 @@ class DirCallback : public AFS::TraverserCallback { public: DirCallback(TraverserConfig& cfg, - const Zstring& parentRelPathPf, //postfixed with FILE_NAME_SEPARATOR! + Zstring&& parentRelPathPf, //postfixed with FILE_NAME_SEPARATOR (or empty!) FolderContainer& output, int level) : cfg_(cfg), - parentRelPathPf_(parentRelPathPf), + parentRelPathPf_(std::move(parentRelPathPf)), output_(output), level_(level) {} //MUST NOT use cfg_ during construction! see BaseDirCallback() @@ -289,18 +289,7 @@ void DirCallback::onFile(const AFS::FileInfo& fi) //throw ThreadStopRequest //apply filter before processing (use relative name!) if (!cfg_.filter.ref().passFileFilter(relPath)) return; - - //sync.ffs_db database and lock files are excluded via filter! - - // std::string fileId = details.fileSize >= 1024 * 1024U ? util::retrieveFileID(filepath) : std::string(); - - /* Perf test Windows 7, SSD, 350k files, 50k dirs, files > 1MB: 7000 - regular: 6.9s - ID per file: 43.9s - ID per file > 1MB: 7.2s - ID per dir: 8.4s - - Linux: retrieveFileID takes about 50% longer in VM! (avoidable because of redundant stat() call!) */ + //note: sync.ffs_db database and lock files are excluded via path filter! output_.addSubFile(fi.itemName, FileAttributes(fi.modTime, fi.fileSize, fi.filePrint, fi.isFollowedSymlink)); @@ -312,7 +301,7 @@ std::shared_ptr<AFS::TraverserCallback> DirCallback::onFolder(const AFS::FolderI { interruptionPoint(); //throw ThreadStopRequest - const Zstring& relPath = parentRelPathPf_ + fi.itemName; + Zstring relPath = parentRelPathPf_ + fi.itemName; //update status information no matter if item is excluded or not! if (cfg_.acb.mayReportCurrentFile(cfg_.threadIdx, cfg_.lastReportTime)) @@ -324,7 +313,7 @@ std::shared_ptr<AFS::TraverserCallback> DirCallback::onFolder(const AFS::FolderI const bool passFilter = cfg_.filter.ref().passDirFilter(relPath, &childItemMightMatch); if (!passFilter && !childItemMightMatch) return nullptr; //do NOT traverse subdirs - //else: attention! ensure directory filtering is applied later to exclude actually filtered directories + //else: ensure directory filtering is applied later to exclude actually filtered directories!!! FolderContainer& subFolder = output_.addSubFolder(fi.itemName, FolderAttributes(fi.isFollowedSymlink)); if (passFilter) @@ -343,7 +332,7 @@ std::shared_ptr<AFS::TraverserCallback> DirCallback::onFolder(const AFS::FolderI return nullptr; } - return std::make_shared<DirCallback>(cfg_, relPath + FILE_NAME_SEPARATOR, subFolder, level_ + 1); + return std::make_shared<DirCallback>(cfg_, std::move(relPath += FILE_NAME_SEPARATOR), subFolder, level_ + 1); } @@ -434,7 +423,7 @@ std::map<DirectoryKey, DirectoryValue> fff::parallelDeviceTraversal(const std::s for (const auto& [afsDevice, dirKeys] : perDeviceFolders) { const int threadIdx = static_cast<int>(worker.size()); - Zstring threadName = Zstr("Comp Device[") + numberTo<Zstring>(threadIdx + 1) + Zstr('/') + numberTo<Zstring>(perDeviceFolders.size()) + Zstr("] ") + + Zstring threadName = Zstr("Compare[") + numberTo<Zstring>(threadIdx + 1) + Zstr('/') + numberTo<Zstring>(perDeviceFolders.size()) + Zstr("] ") + utfTo<Zstring>(AFS::getDisplayPath({afsDevice, AfsPath()})); const size_t parallelOps = 1; diff --git a/FreeFileSync/Source/base/process_callback.h b/FreeFileSync/Source/base/process_callback.h index 417fcf01..5bd75439 100644 --- a/FreeFileSync/Source/base/process_callback.h +++ b/FreeFileSync/Source/base/process_callback.h @@ -40,8 +40,14 @@ struct PhaseCallback //UI info only, should *not* be logged: called periodically after data was processed: expected(!) to request GUI update virtual void updateStatus(std::wstring&& msg) = 0; //throw X + enum class MsgType + { + info, + warning, + error, + }; //log only; must *not* call updateStatus()! - virtual void logInfo(const std::wstring& msg) = 0; //throw X + virtual void logMessage(const std::wstring& msg, MsgType type) = 0; //throw X virtual void reportWarning(const std::wstring& msg, bool& warningActive) = 0; //throw X @@ -57,6 +63,7 @@ struct PhaseCallback retry }; virtual Response reportError(const ErrorInfo& errorInfo) = 0; //throw X; recoverable error + virtual void reportFatalError(const std::wstring& msg) = 0; //throw X; non-recoverable error }; diff --git a/FreeFileSync/Source/base/status_handler_impl.h b/FreeFileSync/Source/base/status_handler_impl.h index 487e8605..6f7e1a34 100644 --- a/FreeFileSync/Source/base/status_handler_impl.h +++ b/FreeFileSync/Source/base/status_handler_impl.h @@ -49,24 +49,18 @@ public: //blocking call: context of worker thread //=> indirect support for "pause": logInfo() is called under singleThread lock, // so all other worker threads will wait when coming out of parallel I/O (trying to lock singleThread) - void logInfo(const std::wstring& msg) //throw ThreadStopRequest + void logMessage(const std::wstring& msg, PhaseCallback::MsgType type) //throw ThreadStopRequest { assert(!zen::runningOnMainThread()); - std::unique_lock dummy(lockRequest_); - zen::interruptibleWait(conditionReadyForNewRequest_, dummy, [this] { return !logInfoRequest_; }); //throw ThreadStopRequest - - logInfoRequest_ = /*std::move(taskPrefix) + */ msg; + { + std::unique_lock dummy(lockRequest_); + zen::interruptibleWait(conditionReadyForNewRequest_, dummy, [this] { return !logMsgRequest_; }); //throw ThreadStopRequest - dummy.unlock(); //optimization for condition_variable::notify_all() + logMsgRequest_ = LogMsgRequest{msg, type}; + } conditionNewRequest.notify_all(); } - void reportInfo(std::wstring&& msg) //throw ThreadStopRequest - { - logInfo(msg); //throw ThreadStopRequest - updateStatus(std::move(msg)); // - } - //blocking call: context of worker thread PhaseCallback::Response reportError(const PhaseCallback::ErrorInfo& errorInfo) //throw ThreadStopRequest { @@ -89,6 +83,27 @@ public: return rv; } + //blocking call: context of worker thread + void reportWarning(const std::wstring& msg, bool& warningActive) //throw ThreadStopRequest + { + assert(!zen::runningOnMainThread()); + { + std::unique_lock dummy(lockRequest_); + zen::interruptibleWait(conditionReadyForNewRequest_, dummy, [this] { return !warningRequest_ && !warningResponse_; }); //throw ThreadStopRequest + + warningRequest_ = WarningRequest{msg, warningActive}; + conditionNewRequest.notify_all(); + + zen::interruptibleWait(conditionHaveResponse_, dummy, [this] { return static_cast<bool>(warningResponse_); }); //throw ThreadStopRequest + + warningActive = warningResponse_->warningActive; + + warningRequest_ = std::nullopt; + warningResponse_ = std::nullopt; + } + conditionReadyForNewRequest_.notify_all(); //=> spurious wake-up for AsyncCallback::logInfo() + } + //context of main thread void waitUntilDone(std::chrono::milliseconds cbInterval, PhaseCallback& cb) //throw X { @@ -99,21 +114,32 @@ public: for (std::unique_lock dummy(lockRequest_);;) //process all errors without delay { - const bool rv = conditionNewRequest.wait_until(dummy, callbackTime, [this] { return (errorRequest_ && !errorResponse_) || logInfoRequest_ || finishNowRequest_; }); + const bool rv = conditionNewRequest.wait_until(dummy, callbackTime, [this] + { + return logMsgRequest_ || (errorRequest_ && !errorResponse_) || (warningRequest_ && !warningResponse_) || finishNowRequest_; + }); if (!rv) //time-out + condition not met break; + if (logMsgRequest_) + { + cb.logMessage(logMsgRequest_->msg, logMsgRequest_->type); //throw X + logMsgRequest_ = {}; + conditionReadyForNewRequest_.notify_all(); //=> spurious wake-up for AsyncCallback::reportError() + } if (errorRequest_ && !errorResponse_) { assert(!finishNowRequest_); errorResponse_ = cb.reportError(*errorRequest_); //throw X conditionHaveResponse_.notify_all(); //instead of notify_one(); work around bug: https://svn.boost.org/trac/boost/ticket/7796 } - if (logInfoRequest_) + if (warningRequest_ && !warningResponse_) { - cb.logInfo(*logInfoRequest_); //throw X - logInfoRequest_ = {}; - conditionReadyForNewRequest_.notify_all(); //=> spurious wake-up for AsyncCallback::reportError() + assert(!finishNowRequest_); + bool warningActive = warningRequest_->warningActive; + cb.reportWarning(warningRequest_->msg, warningActive); //throw X + warningResponse_ = WarningResponse{warningActive}; + conditionHaveResponse_.notify_all(); } if (finishNowRequest_) { @@ -175,10 +201,12 @@ public: void notifyAllDone() //noexcept { - std::lock_guard dummy(lockRequest_); - assert(!finishNowRequest_); - finishNowRequest_ = true; - conditionNewRequest.notify_all(); //perf: should unlock mutex before notify!? (insignificant) + { + std::lock_guard dummy(lockRequest_); + assert(!finishNowRequest_); + finishNowRequest_ = true; + } + conditionNewRequest.notify_all(); } private: @@ -263,14 +291,28 @@ private: return statusMsg; } + struct LogMsgRequest + { + std::wstring msg; + PhaseCallback::MsgType type = PhaseCallback::MsgType::error; + }; + struct WarningRequest + { + std::wstring msg; + bool warningActive = false; + }; + struct WarningResponse { bool warningActive = false; }; + //---- main <-> worker communication channel ---- std::mutex lockRequest_; std::condition_variable conditionReadyForNewRequest_; std::condition_variable conditionNewRequest; std::condition_variable conditionHaveResponse_; + std::optional<LogMsgRequest> logMsgRequest_; std::optional<PhaseCallback::ErrorInfo> errorRequest_; std::optional<PhaseCallback::Response > errorResponse_; - std::optional<std::wstring> logInfoRequest_; + std::optional<WarningRequest> warningRequest_; + std::optional<WarningResponse> warningResponse_; bool finishNowRequest_ = false; //---- status updates ---- @@ -311,6 +353,10 @@ public: void updateStatus(std::wstring&& msg) { cb_.updateStatus(std::move(msg)); } //throw X + void logMessage(const std::wstring& msg, PhaseCallback::MsgType type) { cb_.logMessage(msg, type); } //throw X + + void reportWarning(const std::wstring& msg, bool& warningActive) { cb_.reportWarning(msg, warningActive); }//throw X + void reportDelta(int itemsDelta, int64_t bytesDelta) //noexcept! { cb_.updateDataProcessed(itemsDelta, bytesDelta); //noexcept! @@ -330,9 +376,6 @@ public: } } - int64_t getBytesReported() const { return bytesReported_; } - int64_t getBytesExpected() const { return bytesExpected_; } - private: int itemsReported_ = 0; int64_t bytesReported_ = 0; @@ -354,38 +397,35 @@ constexpr std::chrono::seconds STATUS_PERCENT_SPEED_WINDOW(10); template <class Callback> struct PercentStatReporter { - PercentStatReporter(std::wstring&& statusMsg, int64_t bytesExpected, Callback& cb) : //throw X + PercentStatReporter(const std::wstring& statusMsg, int64_t bytesExpected, ItemStatReporter<Callback>& statReporter) : msgPrefix_(statusMsg + L"... "), - statReporter_(1 /*itemsExpected*/, bytesExpected, cb) - { - statReporter_.updateStatus(std::move(statusMsg)); //throw X - } + bytesExpected_(bytesExpected), + statReporter_(statReporter) {} + //[!] no "updateStatus() /*throw X*/" in constructor! let caller decide - void updateStatus(int itemsDelta, int64_t bytesDelta) //throw X + void updateDeltaAndStatus(int64_t bytesDelta) //throw X { - statReporter_.reportDelta(itemsDelta, bytesDelta); + statReporter_.reportDelta(0 /*itemsDelta*/, bytesDelta); + bytesCopied_ += bytesDelta; const auto now = std::chrono::steady_clock::now(); if (now >= lastUpdate_ + UI_UPDATE_INTERVAL / 2) //every ~50 ms { lastUpdate_ = now; - const int64_t bytesCopied = statReporter_.getBytesReported(); - const int64_t bytesTotal = statReporter_.getBytesExpected(); - - if (!showPercent_ && bytesCopied > 0) + if (!showPercent_ && bytesCopied_ > 0) { if (startTime_ == std::chrono::steady_clock::time_point()) { startTime_ = now; //get higher-quality perf stats when starting timing here rather than constructor!? - speedTest_.addSample(std::chrono::seconds(0), 0 /*itemsCurrent*/, bytesCopied); + speedTest_.addSample(std::chrono::seconds(0), 0 /*itemsCurrent*/, bytesCopied_); } else if (const std::chrono::nanoseconds elapsed = now - startTime_; elapsed >= STATUS_PERCENT_DELAY) { - speedTest_.addSample(elapsed, 0 /*itemsCurrent*/, bytesCopied); + speedTest_.addSample(elapsed, 0 /*itemsCurrent*/, bytesCopied_); - if (const std::optional<double> remSecs = speedTest_.getRemainingSec(0, bytesTotal - bytesCopied)) + if (const std::optional<double> remSecs = speedTest_.getRemainingSec(0 /*itemsRemaining*/, bytesExpected_ - bytesCopied_)) if (*remSecs > std::chrono::duration<double>(STATUS_PERCENT_MIN_DURATION).count()) { showPercent_ = true; @@ -395,17 +435,15 @@ struct PercentStatReporter } if (showPercent_) { - speedTest_.addSample(now - startTime_, 0 /*itemsCurrent*/, bytesCopied); + speedTest_.addSample(now - startTime_, 0 /*itemsCurrent*/, bytesCopied_); const std::optional<double> bps = speedTest_.getBytesPerSec(); - statReporter_.updateStatus(msgPrefix_ + formatPercent(std::min(static_cast<double>(bytesCopied) / bytesTotal, 1.0), //> 100% possible! see process_callback.h notes - bps ? *bps : 0, bytesTotal)); //throw X + statReporter_.updateStatus(msgPrefix_ + formatPercent(std::min(static_cast<double>(bytesCopied_) / bytesExpected_, 1.0), //> 100% possible! see process_callback.h notes + bps ? *bps : 0, bytesExpected_)); //throw X } } } - void updateStatus(std::wstring&& msg) { statReporter_.updateStatus(std::move(msg)); } //throw X - private: static std::wstring formatPercent(double fraction, double bytesPerSec, int64_t bytesTotal) { @@ -424,18 +462,26 @@ private: bool showPercent_ = false; const std::wstring msgPrefix_; + const int64_t bytesExpected_; + int64_t bytesCopied_ = 0; std::chrono::steady_clock::time_point startTime_; std::chrono::steady_clock::time_point lastUpdate_; SpeedTest speedTest_{STATUS_PERCENT_SPEED_WINDOW}; - ItemStatReporter<Callback> statReporter_; + ItemStatReporter<Callback>& statReporter_; }; -using AsyncPercentStatReporter = PercentStatReporter<AsyncCallback>; - //===================================================================================================================== +template <class Callback> inline +void reportInfo(std::wstring&& msg, Callback& cb /*throw X*/) //throw X +{ + cb.logMessage(msg, PhaseCallback::MsgType::info); //throw X + cb.updateStatus(std::move(msg)); // +} + + template <class Function, class Callback> inline //return ignored error message if available -std::wstring tryReportingError(Function cmd /*throw FileError*/, Callback& cb /*throw X*/) +std::wstring tryReportingError(Function cmd /*throw FileError*/, Callback& cb /*throw X*/) //throw X { for (size_t retryNumber = 0;; ++retryNumber) try diff --git a/FreeFileSync/Source/base/structures.h b/FreeFileSync/Source/base/structures.h index 7c95e41f..17999e1d 100644 --- a/FreeFileSync/Source/base/structures.h +++ b/FreeFileSync/Source/base/structures.h @@ -175,7 +175,7 @@ inline bool effectivelyEqual(const CompConfig& lhs, const CompConfig& rhs) { return lhs == rhs; } //no change in behavior -enum class DeletionPolicy +enum class DeletionVariant { permanent, recycler, @@ -194,7 +194,7 @@ struct SyncConfig //sync direction settings SyncDirectionConfig directionCfg; - DeletionPolicy handleDeletion = DeletionPolicy::recycler; //use Recycle Bin, delete permanently or move to user-defined location + DeletionVariant deletionVariant = DeletionVariant::recycler; //use Recycle Bin, delete permanently or move to user-defined location //versioning options Zstring versioningFolderPhrase; @@ -211,7 +211,7 @@ inline bool operator==(const SyncConfig& lhs, const SyncConfig& rhs) { return lhs.directionCfg == rhs.directionCfg && - lhs.handleDeletion == rhs.handleDeletion && //!= DeletionPolicy::versioning => still consider versioningFolderPhrase: e.g. user temporarily + lhs.deletionVariant == rhs.deletionVariant && //!= DeletionVariant::versioning => still consider versioningFolderPhrase: e.g. user temporarily lhs.versioningFolderPhrase == rhs.versioningFolderPhrase && //switched to "permanent" deletion and accidentally saved cfg => versioning folder is easily restored lhs.versioningStyle == rhs.versioningStyle && (lhs.versioningStyle == VersioningStyle::replace || @@ -229,8 +229,8 @@ inline bool effectivelyEqual(const SyncConfig& lhs, const SyncConfig& rhs) { return effectivelyEqual(lhs.directionCfg, rhs.directionCfg) && - lhs.handleDeletion == rhs.handleDeletion && - (lhs.handleDeletion != DeletionPolicy::versioning || //only evaluate versioning folder if required! + lhs.deletionVariant == rhs.deletionVariant && + (lhs.deletionVariant != DeletionVariant::versioning || //only evaluate versioning folder if required! ( lhs.versioningFolderPhrase == rhs.versioningFolderPhrase && lhs.versioningStyle == rhs.versioningStyle && @@ -388,7 +388,7 @@ size_t getDeviceParallelOps(const std::map<AfsDevice, size_t>& deviceParallelOps void setDeviceParallelOps( std::map<AfsDevice, size_t>& deviceParallelOps, const Zstring& folderPathPhrase, size_t parallelOps); -std::optional<CompareVariant> getCompVariant(const MainConfiguration& mainCfg); +std::optional<CompareVariant> getCompVariant(const MainConfiguration& mainCfg); std::optional<SyncVariant> getSyncVariant(const MainConfiguration& mainCfg); @@ -401,7 +401,6 @@ struct WarningDialogs bool warnSignificantDifference = true; bool warnNotEnoughDiskSpace = true; bool warnUnresolvedConflicts = true; - bool warnModificationTimeError = true; bool warnRecyclerMissing = true; bool warnInputFieldEmpty = true; bool warnDirectoryLockFailed = true; diff --git a/FreeFileSync/Source/base/synchronization.cpp b/FreeFileSync/Source/base/synchronization.cpp index 7aa49f2e..4ca101af 100644 --- a/FreeFileSync/Source/base/synchronization.cpp +++ b/FreeFileSync/Source/base/synchronization.cpp @@ -29,7 +29,6 @@ using namespace fff; namespace { const size_t CONFLICTS_PREVIEW_MAX = 25; //=> consider memory consumption, log file size, email size! -const size_t MODTIME_ERRORS_PREVIEW_MAX = 25; inline @@ -95,17 +94,14 @@ void SyncStatistics::processFile(const FilePair& file) case SO_DELETE_LEFT: ++deleteLeft_; - physicalDeleteLeft_ = true; break; case SO_DELETE_RIGHT: ++deleteRight_; - physicalDeleteRight_ = true; break; case SO_MOVE_LEFT_TO: ++updateLeft_; - //physicalDeleteLeft_ ? -> usually, no; except when falling back to "copy + delete" break; case SO_MOVE_RIGHT_TO: @@ -119,13 +115,11 @@ void SyncStatistics::processFile(const FilePair& file) case SO_OVERWRITE_LEFT: ++updateLeft_; bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::right>()); - physicalDeleteLeft_ = true; break; case SO_OVERWRITE_RIGHT: ++updateRight_; bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::left>()); - physicalDeleteRight_ = true; break; case SO_UNRESOLVED_CONFLICT: @@ -164,24 +158,20 @@ void SyncStatistics::processLink(const SymlinkPair& symlink) case SO_DELETE_LEFT: ++deleteLeft_; - physicalDeleteLeft_ = true; break; case SO_DELETE_RIGHT: ++deleteRight_; - physicalDeleteRight_ = true; break; case SO_OVERWRITE_LEFT: case SO_COPY_METADATA_TO_LEFT: ++updateLeft_; - physicalDeleteLeft_ = true; break; case SO_OVERWRITE_RIGHT: case SO_COPY_METADATA_TO_RIGHT: ++updateRight_; - physicalDeleteRight_ = true; break; case SO_UNRESOLVED_CONFLICT: @@ -218,12 +208,10 @@ void SyncStatistics::processFolder(const FolderPair& folder) case SO_DELETE_LEFT: //if deletion variant == versioning with user-defined directory existing on other volume, this results in a full copy + delete operation! ++deleteLeft_; //however we cannot (reliably) anticipate this situation, fortunately statistics can be adapted during sync! - physicalDeleteLeft_ = true; break; case SO_DELETE_RIGHT: ++deleteRight_; - physicalDeleteRight_ = true; break; case SO_UNRESOLVED_CONFLICT: @@ -257,9 +245,9 @@ void SyncStatistics::processFolder(const FolderPair& folder) } -/* DeletionPolicy::permanent: deletion frees space - DeletionPolicy::recycler: won't free space until recycler is full, but then frees space - DeletionPolicy::versioning: depends on whether versioning folder is on a different volume +/* DeletionVariant::permanent: deletion frees space + DeletionVariant::recycler: won't free space until recycler is full, but then frees space + DeletionVariant::versioning: depends on whether versioning folder is on a different volume -> if deleted item is a followed symlink, no space is freed -> created/updated/deleted item may be on a different volume than base directory: consider symlinks, junctions! @@ -383,7 +371,7 @@ std::vector<FolderPairSyncCfg> fff::extractSyncCfg(const MainConfiguration& main syncCfg.directionCfg.var, syncCfg.directionCfg.var == SyncVariant::twoWay || detectMovedFilesEnabled(syncCfg.directionCfg), - syncCfg.handleDeletion, + syncCfg.deletionVariant, syncCfg.versioningFolderPhrase, syncCfg.versioningStyle, syncCfg.versionMaxAgeDays, @@ -651,6 +639,9 @@ void getPathRaceCondition(const BaseFolderPair& baseFolderP, const BaseFolderPai //################################################################################################################# +warn_static("review: does flushFileBuffers() make sense?") +//https://stackoverflow.com/questions/67620715/how-to-flush-buffered-data-after-copyfileex + //--------------------- data verification ------------------------- void flushFileBuffers(const Zstring& nativeFilePath) //throw FileError { @@ -696,55 +687,59 @@ void verifyFiles(const AbstractPath& sourcePath, const AbstractPath& targetPath, namespace parallel { inline -AFS::ItemType getItemType(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ return parallelScope([ap] { return AFS::getItemType(ap); /*throw FileError*/ }, singleThread); } +AFS::ItemType getItemType(const AbstractPath& itemPath, std::mutex& singleThread) //throw FileError +{ return parallelScope([itemPath] { return AFS::getItemType(itemPath); /*throw FileError*/ }, singleThread); } inline -std::optional<AFS::ItemType> itemStillExists(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ return parallelScope([ap] { return AFS::itemStillExists(ap); /*throw FileError*/ }, singleThread); } +std::optional<AFS::ItemType> itemStillExists(const AbstractPath& itemPath, std::mutex& singleThread) //throw FileError +{ return parallelScope([itemPath] { return AFS::itemStillExists(itemPath); /*throw FileError*/ }, singleThread); } inline -void removeFileIfExists(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ parallelScope([ap] { AFS::removeFileIfExists(ap); /*throw FileError*/ }, singleThread); } +void removeFileIfExists(const AbstractPath& filePath, std::mutex& singleThread) //throw FileError +{ parallelScope([filePath] { AFS::removeFileIfExists(filePath); /*throw FileError*/ }, singleThread); } inline -void removeSymlinkIfExists(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ parallelScope([ap] { AFS::removeSymlinkIfExists(ap); /*throw FileError*/ }, singleThread); } +void removeSymlinkIfExists(const AbstractPath& linkPath, std::mutex& singleThread) //throw FileError +{ parallelScope([linkPath] { AFS::removeSymlinkIfExists(linkPath); /*throw FileError*/ }, singleThread); } inline void moveAndRenameItem(const AbstractPath& pathFrom, const AbstractPath& pathTo, std::mutex& singleThread) //throw FileError, ErrorMoveUnsupported { parallelScope([pathFrom, pathTo] { AFS::moveAndRenameItem(pathFrom, pathTo); /*throw FileError, ErrorMoveUnsupported*/ }, singleThread); } inline -AbstractPath getSymlinkResolvedPath(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ return parallelScope([ap] { return AFS::getSymlinkResolvedPath(ap); /*throw FileError*/ }, singleThread); } +AbstractPath getSymlinkResolvedPath(const AbstractPath& linkPath, std::mutex& singleThread) //throw FileError +{ return parallelScope([linkPath] { return AFS::getSymlinkResolvedPath(linkPath); /*throw FileError*/ }, singleThread); } + +inline +void copySymlink(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions, std::mutex& singleThread) //throw FileError +{ parallelScope([sourcePath, targetPath, copyFilePermissions] { AFS::copySymlink(sourcePath, targetPath, copyFilePermissions); /*throw FileError*/ }, singleThread); } inline -void copySymlink(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions, std::mutex& singleThread) //throw FileError -{ parallelScope([apSource, apTarget, copyFilePermissions] { AFS::copySymlink(apSource, apTarget, copyFilePermissions); /*throw FileError*/ }, singleThread); } +void copyNewFolder(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions, std::mutex& singleThread) //throw FileError +{ parallelScope([sourcePath, targetPath, copyFilePermissions] { AFS::copyNewFolder(sourcePath, targetPath, copyFilePermissions); /*throw FileError*/ }, singleThread); } inline -void copyNewFolder(const AbstractPath& apSource, const AbstractPath& apTarget, bool copyFilePermissions, std::mutex& singleThread) //throw FileError -{ parallelScope([apSource, apTarget, copyFilePermissions] { AFS::copyNewFolder(apSource, apTarget, copyFilePermissions); /*throw FileError*/ }, singleThread); } +void removeFilePlain(const AbstractPath& filePath, std::mutex& singleThread) //throw FileError +{ parallelScope([filePath] { AFS::removeFilePlain(filePath); /*throw FileError*/ }, singleThread); } inline -void removeFilePlain(const AbstractPath& ap, std::mutex& singleThread) //throw FileError -{ parallelScope([ap] { AFS::removeFilePlain(ap); /*throw FileError*/ }, singleThread); } +std::unique_ptr<AFS::RecycleSession> createRecyclerSession(const AbstractPath& folderPath, std::mutex& singleThread) //throw FileError, RecycleBinUnavailable +{ return parallelScope([folderPath] { return AFS::createRecyclerSession(folderPath); /*throw FileError, RecycleBinUnavailable*/ }, singleThread); } //-------------------------------------------------------------- //ATTENTION CALLBACKS: they also run asynchronously *outside* the singleThread lock! //-------------------------------------------------------------- inline -void removeFolderIfExistsRecursion(const AbstractPath& ap, //throw FileError +void removeFolderIfExistsRecursion(const AbstractPath& folderPath, //throw FileError const std::function<void (const std::wstring& displayPath)>& onBeforeFileDeletion, //optional const std::function<void (const std::wstring& displayPath)>& onBeforeFolderDeletion, //one call for each object! std::mutex& singleThread) -{ parallelScope([ap, onBeforeFileDeletion, onBeforeFolderDeletion] { AFS::removeFolderIfExistsRecursion(ap, onBeforeFileDeletion, onBeforeFolderDeletion); /*throw FileError*/ }, singleThread); } +{ parallelScope([folderPath, onBeforeFileDeletion, onBeforeFolderDeletion] { AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeFolderDeletion); /*throw FileError*/ }, singleThread); } inline -AFS::FileCopyResult copyFileTransactional(const AbstractPath& apSource, const AFS::StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X - const AbstractPath& apTarget, +AFS::FileCopyResult copyFileTransactional(const AbstractPath& sourcePath, const AFS::StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X + const AbstractPath& targetPath, bool copyFilePermissions, bool transactionalCopy, const std::function<void()>& onDeleteTargetFile /*throw X*/, @@ -753,21 +748,21 @@ AFS::FileCopyResult copyFileTransactional(const AbstractPath& apSource, const AF { return parallelScope([=] { - return AFS::copyFileTransactional(apSource, attrSource, apTarget, copyFilePermissions, transactionalCopy, onDeleteTargetFile, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X + return AFS::copyFileTransactional(sourcePath, attrSource, targetPath, copyFilePermissions, transactionalCopy, onDeleteTargetFile, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X }, singleThread); } -inline //RecycleSession::recycleItemIfExists() is internally synchronized! -void recycleItemIfExists(AFS::RecycleSession& recyclerSession, const AbstractPath& ap, const Zstring& logicalRelPath, std::mutex& singleThread) //throw FileError -{ parallelScope([=, &recyclerSession] { return recyclerSession.recycleItemIfExists(ap, logicalRelPath); /*throw FileError*/ }, singleThread); } +inline //RecycleSession::moveToRecycleBin() is internally synchronized! +void moveToRecycleBinIfExists(AFS::RecycleSession& recyclerSession, const AbstractPath& itemPath, const Zstring& logicalRelPath, std::mutex& singleThread) //throw FileError, RecycleBinUnavailable +{ parallelScope([=, &recyclerSession] { return recyclerSession.moveToRecycleBinIfExists(itemPath, logicalRelPath); /*throw FileError, RecycleBinUnavailable*/ }, singleThread); } inline //FileVersioner::revisionFile() is internally synchronized! void revisionFile(FileVersioner& versioner, const FileDescriptor& fileDescr, const Zstring& relativePath, const IoCallback& notifyUnbufferedIO /*throw X*/, std::mutex& singleThread) //throw FileError, X -{ parallelScope([=, &versioner] { return versioner.revisionFile(fileDescr, relativePath, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); } +{ parallelScope([=, &versioner] { versioner.revisionFile(fileDescr, relativePath, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); } inline //FileVersioner::revisionSymlink() is internally synchronized! void revisionSymlink(FileVersioner& versioner, const AbstractPath& linkPath, const Zstring& relativePath, std::mutex& singleThread) //throw FileError -{ parallelScope([=, &versioner] { return versioner.revisionSymlink(linkPath, relativePath); /*throw FileError*/ }, singleThread); } +{ parallelScope([=, &versioner] { versioner.revisionSymlink(linkPath, relativePath); /*throw FileError*/ }, singleThread); } inline //FileVersioner::revisionFolder() is internally synchronized! void revisionFolder(FileVersioner& versioner, @@ -779,8 +774,8 @@ void revisionFolder(FileVersioner& versioner, { parallelScope([=, &versioner] { versioner.revisionFolder(folderPath, relativePath, onBeforeFileMove, onBeforeFolderMove, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); } inline -void verifyFiles(const AbstractPath& apSource, const AbstractPath& apTarget, const IoCallback& notifyUnbufferedIO /*throw X*/, std::mutex& singleThread) //throw FileError, X -{ parallelScope([=] { ::verifyFiles(apSource, apTarget, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); } +void verifyFiles(const AbstractPath& sourcePath, const AbstractPath& targetPath, const IoCallback& notifyUnbufferedIO /*throw X*/, std::mutex& singleThread) //throw FileError, X +{ parallelScope([=] { ::verifyFiles(sourcePath, targetPath, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); } } @@ -790,118 +785,114 @@ void verifyFiles(const AbstractPath& apSource, const AbstractPath& apTarget, con class DeletionHandler //abstract deletion variants: permanently, recycle bin, user-defined directory { public: - DeletionHandler(const AbstractPath& baseFolderPath, //nothrow! - DeletionPolicy deletionPolicy, + DeletionHandler(const AbstractPath& baseFolderPath, + bool& recyclerMissingReportOnce, + bool& warnRecyclerMissing, + DeletionVariant deletionVariant, const AbstractPath& versioningFolderPath, VersioningStyle versioningStyle, - time_t syncStartTime); + time_t syncStartTime); //nothrow! //clean-up temporary directory (recycle bin optimization) void tryCleanup(PhaseCallback& cb /*throw X*/); //throw X - void removeDirWithCallback (const AbstractPath& dirPath, const Zstring& relativePath, AsyncItemStatReporter& statReporter, std::mutex& singleThread); // - void removeFileWithCallback(const FileDescriptor& fileDescr, const Zstring& relativePath, AsyncItemStatReporter& statReporter, std::mutex& singleThread); //throw FileError, ThreadStopRequest - void removeLinkWithCallback(const AbstractPath& linkPath, const Zstring& relativePath, AsyncItemStatReporter& statReporter, std::mutex& singleThread); // - - const std::wstring& getTxtRemovingFile () const { return txtRemovingFile_; } // - const std::wstring& getTxtRemovingFolder () const { return txtRemovingFolder_; } //buffered status texts - const std::wstring& getTxtRemovingSymLink() const { return txtRemovingSymlink_; } // + void removeFileWithCallback(const FileDescriptor& fileDescr, const Zstring& relPath, bool beforeOverwrite, AsyncItemStatReporter& statReporter, std::mutex& singleThread); //throw FileError, ThreadStopRequest + void removeLinkWithCallback(const AbstractPath& linkPath, const Zstring& relPath, bool beforeOverwrite, AsyncItemStatReporter& statReporter, std::mutex& singleThread); // + void removeDirWithCallback (const AbstractPath& dirPath, const Zstring& relPath, AsyncItemStatReporter& statReporter, std::mutex& singleThread); // private: DeletionHandler (const DeletionHandler&) = delete; DeletionHandler& operator=(const DeletionHandler&) = delete; - AFS::RecycleSession& getOrCreateRecyclerSession() //throw FileError => dont create in constructor!!! + //might not be needed => create lazily: + AFS::RecycleSession& getOrCreateRecyclerSession(std::mutex& singleThread) //throw FileError, RecycleBinUnavailable { - assert(deletionPolicy_ == DeletionPolicy::recycler); - if (!recyclerSession_) - recyclerSession_ = AFS::createRecyclerSession(baseFolderPath_); //throw FileError - return *recyclerSession_; + assert(deletionVariant_ == DeletionVariant::recycler); + + if (!recyclerSession_ && !recyclerUnavailableExcept_) + try + { + recyclerSession_ = parallel::createRecyclerSession(baseFolderPath_, singleThread); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) { recyclerUnavailableExcept_ = e; } + + if (recyclerUnavailableExcept_) + throw* recyclerUnavailableExcept_; //throw RecycleBinUnavailable + else + return *recyclerSession_; } - FileVersioner& getOrCreateVersioner() //throw FileError => dont create in constructor!!! + //might not be needed => create lazily: + FileVersioner& getOrCreateVersioner() //throw FileError { - assert(deletionPolicy_ == DeletionPolicy::versioning); + assert(deletionVariant_ == DeletionVariant::versioning); if (!versioner_) versioner_ = std::make_unique<FileVersioner>(versioningFolderPath_, versioningStyle_, syncStartTime_); //throw FileError return *versioner_; } - const DeletionPolicy deletionPolicy_; //keep it invariant! e.g. consider getOrCreateVersioner() one-time construction! + bool& recyclerMissingReportOnce_; //shared by threads! access under "singleThread" lock! + bool& warnRecyclerMissing_; //WarningDialogs::warnRecyclerMissing + + const DeletionVariant deletionVariant_; //keep it invariant! e.g. consider getOrCreateVersioner() one-time construction! const AbstractPath baseFolderPath_; - std::unique_ptr<AFS::RecycleSession> recyclerSession_; - //used only for DeletionPolicy::versioning: + std::unique_ptr<AFS::RecycleSession> recyclerSession_; //it's one of these (or none if not yet initialized) + std::optional<RecycleBinUnavailable> recyclerUnavailableExcept_; // + + //used only for DeletionVariant::versioning: const AbstractPath versioningFolderPath_; const VersioningStyle versioningStyle_; const time_t syncStartTime_; std::unique_ptr<FileVersioner> versioner_; //buffer status texts: - const std::wstring txtRemovingFile_; - const std::wstring txtRemovingSymlink_; - const std::wstring txtRemovingFolder_; + const std::wstring txtDelFilePermanent_ = _("Deleting file %x"); + const std::wstring txtDelFileRecycler_ = _("Moving file %x to the recycle bin"); + const std::wstring txtDelFileVersioning_ = replaceCpy(_("Moving file %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); + + const std::wstring txtDelSymlinkPermanent_ = _("Deleting symbolic link %x"); + const std::wstring txtDelSymlinkRecycler_ = _("Moving symbolic link %x to the recycle bin"); + const std::wstring txtDelSymlinkVersioning_ = replaceCpy(_("Moving symbolic link %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); + + const std::wstring txtDelFolderPermanent_ = _("Deleting folder %x"); + const std::wstring txtDelFolderRecycler_ = _("Moving folder %x to the recycle bin"); + const std::wstring txtDelFolderVersioning_ = replaceCpy(_("Moving folder %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); + const std::wstring txtMovingFileXtoY_ = _("Moving file %x to %y"); const std::wstring txtMovingFolderXtoY_ = _("Moving folder %x to %y"); }; -DeletionHandler::DeletionHandler(const AbstractPath& baseFolderPath, //nothrow! - DeletionPolicy deletionPolicy, +DeletionHandler::DeletionHandler(const AbstractPath& baseFolderPath, + bool& recyclerMissingReportOnce, + bool& warnRecyclerMissing, + DeletionVariant deletionVariant, const AbstractPath& versioningFolderPath, VersioningStyle versioningStyle, time_t syncStartTime) : - deletionPolicy_(deletionPolicy), + recyclerMissingReportOnce_(recyclerMissingReportOnce), + warnRecyclerMissing_(warnRecyclerMissing), + deletionVariant_(deletionVariant), baseFolderPath_(baseFolderPath), versioningFolderPath_(versioningFolderPath), versioningStyle_(versioningStyle), - syncStartTime_(syncStartTime), - //*INDENT-OFF* - txtRemovingFile_([&] - { - switch (deletionPolicy) - { - case DeletionPolicy::permanent: return _("Deleting file %x"); - case DeletionPolicy::recycler: return _("Moving file %x to the recycle bin"); - case DeletionPolicy::versioning: return replaceCpy(_("Moving file %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); - } - return std::wstring(); - }()), - txtRemovingSymlink_([&] - { - switch (deletionPolicy) - { - case DeletionPolicy::permanent: return _("Deleting symbolic link %x"); - case DeletionPolicy::recycler: return _("Moving symbolic link %x to the recycle bin"); - case DeletionPolicy::versioning: return replaceCpy(_("Moving symbolic link %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); - } - return std::wstring(); - }()), - txtRemovingFolder_([&] - { - switch (deletionPolicy) - { - case DeletionPolicy::permanent: return _("Deleting folder %x"); - case DeletionPolicy::recycler: return _("Moving folder %x to the recycle bin"); - case DeletionPolicy::versioning: return replaceCpy(_("Moving folder %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_))); - } - return std::wstring(); - }()) {} - //*INDENT-ON* + syncStartTime_(syncStartTime) {} + void DeletionHandler::tryCleanup(PhaseCallback& cb /*throw X*/) //throw X { assert(runningOnMainThread()); - switch (deletionPolicy_) + switch (deletionVariant_) { - case DeletionPolicy::recycler: + case DeletionVariant::recycler: if (recyclerSession_) { auto notifyDeletionStatus = [&](const std::wstring& displayPath) { if (!displayPath.empty()) - cb.updateStatus(replaceCpy(txtRemovingFile_, L"%x", fmtPath(displayPath))); //throw X + cb.updateStatus(replaceCpy(txtDelFileRecycler_, L"%x", fmtPath(displayPath))); //throw X else cb.requestUiUpdate(); //throw X }; @@ -910,113 +901,191 @@ void DeletionHandler::tryCleanup(PhaseCallback& cb /*throw X*/) //throw X } break; - case DeletionPolicy::permanent: - case DeletionPolicy::versioning: + case DeletionVariant::permanent: + case DeletionVariant::versioning: break; } } -void DeletionHandler::removeDirWithCallback(const AbstractPath& folderPath,//throw FileError, ThreadStopRequest - const Zstring& relativePath, - AsyncItemStatReporter& statReporter, std::mutex& singleThread) +void DeletionHandler::removeFileWithCallback(const FileDescriptor& fileDescr, const Zstring& relPath, bool beforeOverwrite, + AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, ThreadStopRequest { - switch (deletionPolicy_) + if (deletionVariant_ != DeletionVariant::permanent && + endsWith(relPath, AFS::TEMP_FILE_ENDING)) //special rule: always delete .ffs_tmp files permanently! { - case DeletionPolicy::permanent: - { - //callbacks run *outside* singleThread_ lock! => fine - auto notifyDeletion = [&statReporter](const std::wstring& statusText, const std::wstring& displayPath) - { - statReporter.updateStatus(replaceCpy(statusText, L"%x", fmtPath(displayPath))); //throw ThreadStopRequest - statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done! - //OTOH: ThreadStopRequest must not happen just after last deletion was successful: allow for transactional file model update! - }; - static_assert(std::is_const_v<decltype(txtRemovingFile_)>, "callbacks better be thread-safe!"); - auto onBeforeFileDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtRemovingFile_, displayPath); }; - auto onBeforeDirDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtRemovingFolder_, displayPath); }; - - parallel::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeDirDeletion, singleThread); //throw FileError - } - break; - - case DeletionPolicy::recycler: - parallel::recycleItemIfExists(getOrCreateRecyclerSession(), folderPath, relativePath, singleThread); //throw FileError - statReporter.reportDelta(1, 0); //moving to recycler is ONE logical operation, irrespective of the number of child elements! - break; - - case DeletionPolicy::versioning: - { - //callbacks run *outside* singleThread_ lock! => fine - auto notifyMove = [&statReporter](const std::wstring& statusText, const std::wstring& displayPathFrom, const std::wstring& displayPathTo) - { - statReporter.updateStatus(replaceCpy(replaceCpy(statusText, L"%x", L'\n' + fmtPath(displayPathFrom)), L"%y", L'\n' + fmtPath(displayPathTo))); //throw ThreadStopRequest - statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done! - }; - static_assert(std::is_const_v<decltype(txtMovingFileXtoY_)>, "callbacks better be thread-safe!"); - auto onBeforeFileMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFileXtoY_, displayPathFrom, displayPathTo); }; - auto onBeforeFolderMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFolderXtoY_, displayPathFrom, displayPathTo); }; - auto notifyUnbufferedIO = [&](int64_t bytesDelta) { statReporter.reportDelta(0, bytesDelta); interruptionPoint(); }; //throw ThreadStopRequest - - parallel::revisionFolder(getOrCreateVersioner(), folderPath, relativePath, onBeforeFileMove, onBeforeFolderMove, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest - } - break; - } -} - - -void DeletionHandler::removeFileWithCallback(const FileDescriptor& fileDescr, //throw FileError, ThreadStopRequest - const Zstring& relativePath, - AsyncItemStatReporter& statReporter, std::mutex& singleThread) -{ - - if (endsWith(relativePath, AFS::TEMP_FILE_ENDING)) //special rule for .ffs_tmp files: always delete permanently! + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError + } else - switch (deletionPolicy_) + /* don't use AsyncItemStatReporter if "beforeOverwrite": + - logInfo/updateStatus() is superfluous/confuses user, except: do show progress and allow cancel for versioning! + - no (logical) item count update desired + => BUT: total byte count should still be adjusted if versioning requires a file copy instead of a move! + - if fail-safe file copy is active, then the next operation will be a simple "rename" + => don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated! */ + switch (deletionVariant_) { - case DeletionPolicy::permanent: + case DeletionVariant::permanent: + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError break; - case DeletionPolicy::recycler: - parallel::recycleItemIfExists(getOrCreateRecyclerSession(), fileDescr.path, relativePath, singleThread); //throw FileError + + case DeletionVariant::recycler: + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFileRecycler_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest + try + { + parallel::moveToRecycleBinIfExists(getOrCreateRecyclerSession(singleThread), fileDescr.path, relPath, singleThread); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock! + { + recyclerMissingReportOnce_ = true; + statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest + } + if (!beforeOverwrite) statReporter.logMessage(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest + parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError + } break; - case DeletionPolicy::versioning: + + case DeletionVariant::versioning: { - //callback runs *outside* singleThread_ lock! => fine - auto notifyUnbufferedIO = [&](int64_t bytesDelta) { statReporter.reportDelta(0, bytesDelta); interruptionPoint(); }; //throw ThreadStopRequest + std::wstring statusMsg = replaceCpy(txtDelFileVersioning_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))); + PercentStatReporter percentReporter(statusMsg, fileDescr.attr.fileSize, statReporter); + + if (!beforeOverwrite) reportInfo(std::move(statusMsg), statReporter); //throw ThreadStopRequest + //else: 1. versioning is moving only: no (potentially throwing) status updates + // 2. versioning needs to copy: may throw ThreadStopRequest, but *no* status updates, unless copying takes so long that % needs to be displayed - parallel::revisionFile(getOrCreateVersioner(), fileDescr, relativePath, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest + //callback runs *outside* singleThread_ lock! => fine + IoCallback notifyUnbufferedIO = [&](int64_t bytesDelta) + { + percentReporter.updateDeltaAndStatus(bytesDelta); //throw ThreadStopRequest + interruptionPoint(); //throw ThreadStopRequest => not reliably covered by PercentStatReporter::updateDeltaAndStatus()! + }; + parallel::revisionFile(getOrCreateVersioner(), fileDescr, relPath, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest } break; } - //even if the source item does not exist anymore, significant I/O work was done => report - //-> also consider unconditional statReporter.reportDelta(-1, 0) when overwriting a file - statReporter.reportDelta(1, 0); + //even if the source item did not exist anymore, significant I/O work was done => report unconditionally + if (!beforeOverwrite) statReporter.reportDelta(1, 0); } -void DeletionHandler::removeLinkWithCallback(const AbstractPath& linkPath, //throw FileError, throw ThreadStopRequest - const Zstring& relativePath, - AsyncItemStatReporter& statReporter, std::mutex& singleThread) +void DeletionHandler::removeLinkWithCallback(const AbstractPath& linkPath, const Zstring& relPath, bool beforeOverwrite, + AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, throw ThreadStopRequest { - switch (deletionPolicy_) - { - case DeletionPolicy::permanent: + /* don't use AsyncItemStatReporter if "beforeOverwrite": + - logInfo() is superfluous/confuses user + - no (logical) item count update desired + - don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated! */ + switch (deletionVariant_) + { + case DeletionVariant::permanent: + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest parallel::removeSymlinkIfExists(linkPath, singleThread); //throw FileError break; - case DeletionPolicy::recycler: - parallel::recycleItemIfExists(getOrCreateRecyclerSession(), linkPath, relativePath, singleThread); //throw FileError + + case DeletionVariant::recycler: + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkRecycler_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest + try + { + parallel::moveToRecycleBinIfExists(getOrCreateRecyclerSession(singleThread), linkPath, relPath, singleThread); //throw FileError, RecycleBinUnavailable + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock! + { + recyclerMissingReportOnce_ = true; + statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest + } + if (!beforeOverwrite) statReporter.logMessage(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest + parallel::removeSymlinkIfExists(linkPath, singleThread); //throw FileError + } break; - case DeletionPolicy::versioning: - parallel::revisionSymlink(getOrCreateVersioner(), linkPath, relativePath, singleThread); //throw FileError + + case DeletionVariant::versioning: + if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkVersioning_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest + parallel::revisionSymlink(getOrCreateVersioner(), linkPath, relPath, singleThread); //throw FileError break; } //remain transactional as much as possible => no more callbacks that can throw after successful deletion! (next: update file model!) - //report unconditionally, see removeFileWithCallback() - statReporter.reportDelta(1, 0); + //even if the source item did not exist anymore, significant I/O work was done => report unconditionally + if (!beforeOverwrite) statReporter.reportDelta(1, 0); +} + + +void DeletionHandler::removeDirWithCallback(const AbstractPath& folderPath, const Zstring& relPath, + AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, ThreadStopRequest +{ + auto removeFolderPermanently = [&] + { + //callbacks run *outside* singleThread_ lock! => fine + auto notifyDeletion = [&statReporter](const std::wstring& statusText, const std::wstring& displayPath) + { + statReporter.updateStatus(replaceCpy(statusText, L"%x", fmtPath(displayPath))); //throw ThreadStopRequest + statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done! + }; + static_assert(std::is_const_v<decltype(txtDelFilePermanent_)>, "callbacks better be thread-safe!"); + + auto onBeforeFileDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtDelFilePermanent_, displayPath); }; + auto onBeforeDirDeletion = [&](const std::wstring& displayPath) { notifyDeletion(txtDelFolderPermanent_, displayPath); }; + parallel::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeDirDeletion, singleThread); //throw FileError + }; + + switch (deletionVariant_) + { + case DeletionVariant::permanent: + { + reportInfo(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest + removeFolderPermanently(); //throw FileError, ThreadStopRequest + } + break; + + case DeletionVariant::recycler: + reportInfo(replaceCpy(txtDelFolderRecycler_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest + try + { + parallel::moveToRecycleBinIfExists(getOrCreateRecyclerSession(singleThread), folderPath, relPath, singleThread); //throw FileError, RecycleBinUnavailable + statReporter.reportDelta(1, 0); //moving to recycler is ONE logical operation, irrespective of the number of child elements! + } + catch (const RecycleBinUnavailable& e) + { + if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock! + { + recyclerMissingReportOnce_ = true; + statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest + } + statReporter.logMessage(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))) + + L" [" + _("The recycle bin is not available") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest + removeFolderPermanently(); //throw FileError, ThreadStopRequest + } + break; + + case DeletionVariant::versioning: + { + reportInfo(replaceCpy(txtDelFolderVersioning_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest + + //callbacks run *outside* singleThread_ lock! => fine + auto notifyMove = [&statReporter](const std::wstring& statusText, const std::wstring& displayPathFrom, const std::wstring& displayPathTo) + { + statReporter.updateStatus(replaceCpy(replaceCpy(statusText, L"%x", L'\n' + fmtPath(displayPathFrom)), L"%y", L'\n' + fmtPath(displayPathTo))); //throw ThreadStopRequest + statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done! + }; + static_assert(std::is_const_v<decltype(txtMovingFileXtoY_)>, "callbacks better be thread-safe!"); + auto onBeforeFileMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFileXtoY_, displayPathFrom, displayPathTo); }; + auto onBeforeFolderMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFolderXtoY_, displayPathFrom, displayPathTo); }; + auto notifyUnbufferedIO = [&](int64_t bytesDelta) { statReporter.reportDelta(0, bytesDelta); interruptionPoint(); }; //throw ThreadStopRequest + + parallel::revisionFolder(getOrCreateVersioner(), folderPath, relPath, onBeforeFileMove, onBeforeFolderMove, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest + } + break; + } } //=================================================================================================== @@ -1127,7 +1196,6 @@ public: bool verifyCopiedFiles; bool copyFilePermissions; bool failSafeFileCopy; - std::vector<FileError>& errorsModTime; DeletionHandler& delHandlerLeft; DeletionHandler& delHandlerRight; }; @@ -1151,7 +1219,6 @@ private: }; FolderPairSyncer(SyncCtx& syncCtx, std::mutex& singleThread, AsyncCallback& acb) : - errorsModTime_ (syncCtx.errorsModTime), delHandlerLeft_ (syncCtx.delHandlerLeft), delHandlerRight_ (syncCtx.delHandlerRight), verifyCopiedFiles_ (syncCtx.verifyCopiedFiles), @@ -1183,24 +1250,20 @@ private: void synchronizeFolder(FolderPair& folder); // template <SelectSide sideTrg> void synchronizeFolderInt(FolderPair& folder, SyncOperation syncOp); //throw FileError, ThreadStopRequest - void logInfo(const std::wstring& rawText, const std::wstring& displayPath) { acb_.logInfo (replaceCpy(rawText, L"%x", fmtPath(displayPath))); } - void reportInfo(const std::wstring& rawText, const std::wstring& displayPath) { acb_.reportInfo(replaceCpy(rawText, L"%x", fmtPath(displayPath))); } + void reportItemInfo(const std::wstring& msgTemplate, const AbstractPath& itemPath) { reportInfo(replaceCpy(msgTemplate, L"%x", fmtPath(AFS::getDisplayPath(itemPath))), acb_); } - void logInfo(const std::wstring& rawText, const std::wstring& displayPath1, const std::wstring& displayPath2) //throw ThreadStopRequest + void reportItemInfo(const std::wstring& msgTemplate, const AbstractPath& itemPath1, const AbstractPath& itemPath2) //throw ThreadStopRequest { - acb_.logInfo(replaceCpy(replaceCpy(rawText, L"%x", L'\n' + fmtPath(displayPath1)), L"%y", L'\n' + fmtPath(displayPath2))); //throw ThreadStopRequest - } - void reportInfo(const std::wstring& rawText, const std::wstring& displayPath1, const std::wstring& displayPath2) //throw ThreadStopRequest - { - acb_.reportInfo(replaceCpy(replaceCpy(rawText, L"%x", L'\n' + fmtPath(displayPath1)), L"%y", L'\n' + fmtPath(displayPath2))); //throw ThreadStopRequest + reportInfo(replaceCpy(replaceCpy(msgTemplate, L"%x", L'\n' + fmtPath(AFS::getDisplayPath(itemPath1))), + L"%y", L'\n' + fmtPath(AFS::getDisplayPath(itemPath2))), acb_); //throw ThreadStopRequest } //already existing after onDeleteTargetFile(): undefined behavior! (e.g. fail/overwrite/auto-rename) - AFS::FileCopyResult copyFileWithCallback(const FileDescriptor& sourceDescr, //throw FileError, ThreadStopRequest, X + AFS::FileCopyResult copyFileWithCallback(const FileDescriptor& sourceDescr, const AbstractPath& targetPath, const std::function<void()>& onDeleteTargetFile /*throw X*/, //optional! - AsyncPercentStatReporter& statReporter); - std::vector<FileError>& errorsModTime_; + AsyncItemStatReporter& statReporter, //ThreadStopRequest + const std::wstring& statusMsg); //throw FileError, ThreadStopRequest, X DeletionHandler& delHandlerLeft_; DeletionHandler& delHandlerRight_; @@ -1262,7 +1325,7 @@ void FolderPairSyncer::runPass(PassNo pass, SyncCtx& syncCtx, BaseFolderPair& ba ZEN_ON_SCOPE_EXIT( for (InterruptibleThread& wt : worker) wt.requestStop(); ); //stop *all* at the same time before join! size_t threadIdx = 0; - Zstring threadName = Zstr("Sync Worker"); + Zstring threadName = Zstr("Sync"); worker.emplace_back([threadIdx, &singleThread, &acb, &workload, threadName = std::move(threadName)] { setCurrentThreadName(threadName); @@ -1297,10 +1360,6 @@ RingBuffer<Workload::WorkItems> FolderPairSyncer::getFolderLevelWorkItems(PassNo if (pass == PassNo::zero) { - for (FilePair& file : hierObj.refSubFiles()) - if (needZeroPass(file)) - workItems.push_back([this, &file] { executeFileMove(file); /*throw ThreadStopRequest*/ }); - //create folders as required by file move targets: for (FolderPair& folder : hierObj.refSubFolders()) if (needZeroPass(folder) && @@ -1314,15 +1373,19 @@ RingBuffer<Workload::WorkItems> FolderPairSyncer::getFolderLevelWorkItems(PassNo }); else foldersToInspect.push_back(&folder); + + for (FilePair& file : hierObj.refSubFiles()) + if (needZeroPass(file)) + workItems.push_back([this, &file] { executeFileMove(file); /*throw ThreadStopRequest*/ }); } else { - //synchronize folders: + //synchronize folders *first* (see comment above "Multithreaded File Copy") for (FolderPair& folder : hierObj.refSubFolders()) if (pass == getPass(folder)) workItems.push_back([this, &folder, &workload, pass] { - tryReportingError([&] { synchronizeFolder(folder); }, acb_); //throw ThreadStopRequest + tryReportingError([&]{ synchronizeFolder(folder); }, acb_); //throw ThreadStopRequest workload.addWorkItems(getFolderLevelWorkItems(pass, folder, workload)); }); @@ -1334,7 +1397,7 @@ RingBuffer<Workload::WorkItems> FolderPairSyncer::getFolderLevelWorkItems(PassNo if (pass == getPass(file)) workItems.push_back([this, &file] { - tryReportingError([&] { synchronizeFile(file); }, acb_); //throw ThreadStopRequest + tryReportingError([&]{ synchronizeFile(file); }, acb_); //throw ThreadStopRequest }); //synchronize symbolic links: @@ -1393,10 +1456,10 @@ void FolderPairSyncer::executeFileMoveImpl(FilePair& fileFrom, FilePair& fileTo) if (parentMissing) { - logInfo(_("Cannot move file %x to %y.") + L"\n\n" + - replaceCpy(_("Parent folder %x is not existing."), L"%x", fmtPath(AFS::getDisplayPath(parentMissing->getAbstractPath<side>()))), - AFS::getDisplayPath(fileFrom.getAbstractPath<side>()), - AFS::getDisplayPath(fileTo .getAbstractPath<side>())); //throw ThreadStopRequest + reportItemInfo(_("Cannot move file %x to %y.") + L"\n\n" + + replaceCpy(_("Parent folder %x is not existing."), L"%x", fmtPath(AFS::getDisplayPath(parentMissing->getAbstractPath<side>()))), + fileFrom.getAbstractPath<side>(), + fileTo .getAbstractPath<side>()); //throw ThreadStopRequest return true; } @@ -1404,9 +1467,10 @@ void FolderPairSyncer::executeFileMoveImpl(FilePair& fileFrom, FilePair& fileTo) if (haveNameClash(fileTo.getItemNameAny(), fileTo.parent().refSubFolders()) || haveNameClash(fileTo.getItemNameAny(), fileTo.parent().refSubLinks ())) { - logInfo(_("Cannot move file %x to %y.") + L"\n\n" + replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(fileTo.getItemNameAny())), - AFS::getDisplayPath(fileFrom.getAbstractPath<side>()), - AFS::getDisplayPath(fileTo .getAbstractPath<side>())); //throw ThreadStopRequest + reportItemInfo(_("Cannot move file %x to %y.") + L"\n\n" + + replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(fileTo.getItemNameAny())), + fileFrom.getAbstractPath<side>(), + fileTo .getAbstractPath<side>()); //throw ThreadStopRequest return true; } @@ -1419,7 +1483,7 @@ void FolderPairSyncer::executeFileMoveImpl(FilePair& fileFrom, FilePair& fileTo) } catch (const ErrorMoveUnsupported& e) { - acb_.logInfo(e.toString()); //let user know that move operation is not supported, then fall back: + acb_.logMessage(e.toString(), PhaseCallback::MsgType::info); //let user know that move operation is not supported, then fall back: moveSupported = false; } }, acb_); @@ -1702,18 +1766,19 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) const AbstractPath targetPath = file.getAbstractPath<sideTrg>(); - std::wstring statusMsg = replaceCpy(txtCreatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPath))); - acb_.logInfo(statusMsg); //throw ThreadStopRequest - AsyncPercentStatReporter statReporter(std::move(statusMsg), file.getFileSize<sideSrc>(), acb_); //throw ThreadStopRequest + const std::wstring& statusMsg = replaceCpy(txtCreatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPath))); + reportInfo(std::wstring(statusMsg), acb_); //throw ThreadStopRequest + AsyncItemStatReporter statReporter(1, file.getFileSize<sideSrc>(), acb_); try { const AFS::FileCopyResult result = copyFileWithCallback({file.getAbstractPath<sideSrc>(), file.getAttributes<sideSrc>()}, targetPath, nullptr, //onDeleteTargetFile: nothing to delete //if existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - statReporter); //throw FileError, ThreadStopRequest - statReporter.updateStatus(1, 0); //throw ThreadStopRequest + statReporter, + statusMsg); //throw FileError, ThreadStopRequest + statReporter.reportDelta(1, 0); //update FilePair file.setSyncedTo<sideTrg>(file.getItemName<sideSrc>(), result.fileSize, @@ -1723,17 +1788,8 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) result.sourceFilePrint, false, file.isFollowedSymlink<sideSrc>()); - if (result.errorModTime) - switch (file.base().getCompVariant()) - { - case CompareVariant::timeSize: - errorsModTime_.push_back(*result.errorModTime); //show all warnings later as a single message - break; - case CompareVariant::content: //just log, no warning: - case CompareVariant::size: //e.g. FTP server not supporting MFMT command - acb_.logInfo(result.errorModTime->toString()); - break; - } + if (result.errorModTime) //log only; no popup + acb_.logMessage(result.errorModTime->toString(), PhaseCallback::MsgType::warning); //throw ThreadStopRequest } catch (const FileError& e) { @@ -1746,9 +1802,9 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) //do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it! if (!sourceExists) { - logInfo(txtSourceItemNotExist_, AFS::getDisplayPath(file.getAbstractPath<sideSrc>())); //throw ThreadStopRequest + reportItemInfo(txtSourceItemNotExist_, file.getAbstractPath<sideSrc>()); //throw ThreadStopRequest - statReporter.updateStatus(1, 0); //throw ThreadStopRequest + statReporter.reportDelta(1, 0); //even if the source item does not exist anymore, significant I/O work was done => report file.removeObject<sideSrc>(); //source deleted meanwhile...nothing was done (logical point of view!) //remove only *after* evaluating "file, sideSrc"! @@ -1761,15 +1817,14 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) case SO_DELETE_LEFT: case SO_DELETE_RIGHT: - reportInfo(delHandlerTrg.getTxtRemovingFile(), AFS::getDisplayPath(file.getAbstractPath<sideTrg>())); //throw ThreadStopRequest - { - AsyncItemStatReporter statReporter(1, 0, acb_); + { + AsyncItemStatReporter statReporter(1, 0, acb_); - delHandlerTrg.removeFileWithCallback({file.getAbstractPath<sideTrg>(), file.getAttributes<sideTrg>()}, - file.getRelativePath<sideTrg>(), statReporter, singleThread_); //throw FileError, X - file.removeObject<sideTrg>(); //update FilePair - } - break; + delHandlerTrg.removeFileWithCallback({file.getAbstractPath<sideTrg>(), file.getAttributes<sideTrg>()}, file.getRelativePath<sideTrg>(), + false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest + file.removeObject<sideTrg>(); //update FilePair + } + break; case SO_MOVE_LEFT_TO: case SO_MOVE_RIGHT_TO: @@ -1784,7 +1839,7 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) const AbstractPath pathFrom = fileFrom->getAbstractPath<sideTrg>(); const AbstractPath pathTo = fileTo ->getAbstractPath<sideTrg>(); - reportInfo(txtMovingFileXtoY_, AFS::getDisplayPath(pathFrom), AFS::getDisplayPath(pathTo)); //throw ThreadStopRequest + reportItemInfo(txtMovingFileXtoY_, pathFrom, pathTo); //throw ThreadStopRequest AsyncItemStatReporter statReporter(1, 0, acb_); @@ -1819,9 +1874,10 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) if (file.isFollowedSymlink<sideTrg>()) //follow link when updating file rather than delete it and replace with regular file!!! targetPathResolvedOld = targetPathResolvedNew = parallel::getSymlinkResolvedPath(file.getAbstractPath<sideTrg>(), singleThread_); //throw FileError - std::wstring statusMsg = replaceCpy(txtUpdatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPathResolvedOld))); - acb_.logInfo(statusMsg); //throw ThreadStopRequest - AsyncPercentStatReporter statReporter(std::move(statusMsg), file.getFileSize<sideSrc>(), acb_); //throw ThreadStopRequest + const std::wstring& statusMsg = replaceCpy(txtUpdatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPathResolvedOld))); + reportInfo(std::wstring(statusMsg), acb_); //throw ThreadStopRequest + + AsyncItemStatReporter statReporter(1, file.getFileSize<sideSrc>(), acb_); if (file.isFollowedSymlink<sideTrg>()) //since we follow the link, we need to sync case sensitivity of the link manually! if (getUnicodeNormalForm(file.getItemName<sideTrg>()) != @@ -1832,18 +1888,11 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) auto onDeleteTargetFile = [&] //delete target at appropriate time { assert(isLocked(singleThread_)); - //updateStatus(this->delHandlerTrg.getTxtRemovingFile(), AFS::getDisplayPath(targetPathResolvedOld)); -> superfluous/confuses user - FileAttributes followedTargetAttr = file.getAttributes<sideTrg>(); followedTargetAttr.isFollowedSymlink = false; - AsyncItemStatReporter delStatReporter(0, 0, acb_); //=> decouple from AsyncPercentStatReporter above! - //no (logical) item count update desired - but total byte count may change, e.g. move(copy) old file to versioning dir - delHandlerTrg.removeFileWithCallback({targetPathResolvedOld, followedTargetAttr}, file.getRelativePath<sideTrg>(), delStatReporter, singleThread_); //throw FileError, X - delStatReporter.reportDelta(-1, 0); //noexcept! undo item stats reporting within DeletionHandler::removeFileWithCallback() - //if fail-safe file copy is active, then the next operation will be a simple "rename" - //=> don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated! - //=> if failSafeFileCopy_ : don't run callbacks that could throw + delHandlerTrg.removeFileWithCallback({targetPathResolvedOld, followedTargetAttr}, file.getRelativePath<sideTrg>(), + true /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest //file.removeObject<sideTrg>(); -> doesn't make sense for isFollowedSymlink(); "file, sideTrg" evaluated below! }; @@ -1851,8 +1900,9 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) const AFS::FileCopyResult result = copyFileWithCallback({file.getAbstractPath<sideSrc>(), file.getAttributes<sideSrc>()}, targetPathResolvedNew, onDeleteTargetFile, - statReporter); //throw FileError, ThreadStopRequest, X - statReporter.updateStatus(1, 0); //throw ThreadStopRequest + statReporter, + statusMsg); //throw FileError, ThreadStopRequest + statReporter.reportDelta(1, 0); //we model "delete + copy" as ONE logical operation //update FilePair @@ -1864,24 +1914,15 @@ void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) file.isFollowedSymlink<sideTrg>(), file.isFollowedSymlink<sideSrc>()); - if (result.errorModTime) - switch (file.base().getCompVariant()) - { - case CompareVariant::timeSize: - errorsModTime_.push_back(*result.errorModTime); //show all warnings later as a single message - break; - case CompareVariant::content: //just log, no warning: - case CompareVariant::size: //e.g. FTP server not supporting MFMT command - acb_.logInfo(result.errorModTime->toString()); - break; - } + if (result.errorModTime) //log only; no popup + acb_.logMessage(result.errorModTime->toString(), PhaseCallback::MsgType::warning); //throw ThreadStopRequest } break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: //harmonize with file_hierarchy.cpp::getSyncOpDescription!! - reportInfo(txtUpdatingAttributes_, AFS::getDisplayPath(file.getAbstractPath<sideTrg>())); //throw ThreadStopRequest + reportItemInfo(txtUpdatingAttributes_, file.getAbstractPath<sideTrg>()); //throw ThreadStopRequest { AsyncItemStatReporter statReporter(1, 0, acb_); @@ -1957,7 +1998,7 @@ void FolderPairSyncer::synchronizeLinkInt(SymlinkPair& symlink, SyncOperation sy return; //if parent directory creation failed, there's no reason to show more errors! const AbstractPath targetPath = symlink.getAbstractPath<sideTrg>(); - reportInfo(txtCreatingLink_, AFS::getDisplayPath(targetPath)); //throw ThreadStopRequest + reportItemInfo(txtCreatingLink_, targetPath); //throw ThreadStopRequest AsyncItemStatReporter statReporter(1, 0, acb_); try @@ -1982,7 +2023,7 @@ void FolderPairSyncer::synchronizeLinkInt(SymlinkPair& symlink, SyncOperation sy //do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it! if (!sourceExists) { - logInfo(txtSourceItemNotExist_, AFS::getDisplayPath(symlink.getAbstractPath<sideSrc>())); //throw ThreadStopRequest + reportItemInfo(txtSourceItemNotExist_, symlink.getAbstractPath<sideSrc>()); //throw ThreadStopRequest //even if the source item does not exist anymore, significant I/O work was done => report statReporter.reportDelta(1, 0); @@ -1996,47 +2037,44 @@ void FolderPairSyncer::synchronizeLinkInt(SymlinkPair& symlink, SyncOperation sy case SO_DELETE_LEFT: case SO_DELETE_RIGHT: - reportInfo(delHandlerTrg.getTxtRemovingSymLink(), AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>())); //throw ThreadStopRequest - { - AsyncItemStatReporter statReporter(1, 0, acb_); + { + AsyncItemStatReporter statReporter(1, 0, acb_); - delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(), statReporter, singleThread_); //throw FileError, X + delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(), + false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest - symlink.removeObject<sideTrg>(); //update SymlinkPair - } - break; + symlink.removeObject<sideTrg>(); //update SymlinkPair + } + break; case SO_OVERWRITE_LEFT: case SO_OVERWRITE_RIGHT: - reportInfo(txtUpdatingLink_, AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>())); //throw ThreadStopRequest - { - AsyncItemStatReporter statReporter(1, 0, acb_); + { + reportItemInfo(txtUpdatingLink_, symlink.getAbstractPath<sideTrg>()); //throw ThreadStopRequest - //updateStatus(delHandlerTrg.getTxtRemovingSymLink(), AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>())); - delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(), statReporter, singleThread_); //throw FileError, X - statReporter.reportDelta(-1, 0); //undo item stats reporting within DeletionHandler::removeLinkWithCallback() + AsyncItemStatReporter statReporter(1, 0, acb_); - //symlink.removeObject<sideTrg>(); -> "symlink, sideTrg" evaluated below! + delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(), + true /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest - //=> don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated: - //updateStatus(txtUpdatingLink_, AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>())); //restore status text + //symlink.removeObject<sideTrg>(); -> "symlink, sideTrg" evaluated below! - parallel::copySymlink(symlink.getAbstractPath<sideSrc>(), - AFS::appendRelPath(symlink.parent().getAbstractPath<sideTrg>(), symlink.getItemName<sideSrc>()), //respect differences in case of source object - copyFilePermissions_, singleThread_); //throw FileError + parallel::copySymlink(symlink.getAbstractPath<sideSrc>(), + AFS::appendRelPath(symlink.parent().getAbstractPath<sideTrg>(), symlink.getItemName<sideSrc>()), //respect differences in case of source object + copyFilePermissions_, singleThread_); //throw FileError - statReporter.reportDelta(1, 0); //we model "delete + copy" as ONE logical operation + statReporter.reportDelta(1, 0); //we model "delete + copy" as ONE logical operation - //update SymlinkPair - symlink.setSyncedTo<sideTrg>(symlink.getItemName<sideSrc>(), - symlink.getLastWriteTime<sideSrc>(), //target time set from source - symlink.getLastWriteTime<sideSrc>()); - } - break; + //update SymlinkPair + symlink.setSyncedTo<sideTrg>(symlink.getItemName<sideSrc>(), + symlink.getLastWriteTime<sideSrc>(), //target time set from source + symlink.getLastWriteTime<sideSrc>()); + } + break; case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: - reportInfo(txtUpdatingAttributes_, AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>())); //throw ThreadStopRequest + reportItemInfo(txtUpdatingAttributes_, symlink.getAbstractPath<sideTrg>()); //throw ThreadStopRequest { AsyncItemStatReporter statReporter(1, 0, acb_); @@ -2107,7 +2145,7 @@ void FolderPairSyncer::synchronizeFolderInt(FolderPair& folder, SyncOperation sy return; //if parent directory creation failed, there's no reason to show more errors! const AbstractPath targetPath = folder.getAbstractPath<sideTrg>(); - reportInfo(txtCreatingFolder_, AFS::getDisplayPath(targetPath)); //throw ThreadStopRequest + reportItemInfo(txtCreatingFolder_, targetPath); //throw ThreadStopRequest //shallow-"copying" a folder might not fail if source is missing, so we need to check this first: if (parallel::itemStillExists(folder.getAbstractPath<sideSrc>(), singleThread_)) //throw FileError @@ -2137,7 +2175,7 @@ void FolderPairSyncer::synchronizeFolderInt(FolderPair& folder, SyncOperation sy } else //source deleted meanwhile... { - logInfo(txtSourceItemNotExist_, AFS::getDisplayPath(folder.getAbstractPath<sideSrc>())); //throw ThreadStopRequest + reportItemInfo(txtSourceItemNotExist_, folder.getAbstractPath<sideSrc>()); //throw ThreadStopRequest //attention when fixing statistics due to missing folder: child items may be scheduled for move, so deletion will have move-references flip back to copy + delete! const SyncStatistics statsBefore(folder.base()); //=> don't bother considering individual move operations, just calculate over the whole tree @@ -2155,27 +2193,26 @@ void FolderPairSyncer::synchronizeFolderInt(FolderPair& folder, SyncOperation sy case SO_DELETE_LEFT: case SO_DELETE_RIGHT: - reportInfo(delHandlerTrg.getTxtRemovingFolder(), AFS::getDisplayPath(folder.getAbstractPath<sideTrg>())); //throw ThreadStopRequest - { - const SyncStatistics subStats(folder); //counts sub-objects only! - AsyncItemStatReporter statReporter(1 + getCUD(subStats), subStats.getBytesToProcess(), acb_); + { + const SyncStatistics subStats(folder); //counts sub-objects only! + AsyncItemStatReporter statReporter(1 + getCUD(subStats), subStats.getBytesToProcess(), acb_); - delHandlerTrg.removeDirWithCallback(folder.getAbstractPath<sideTrg>(), folder.getRelativePath<sideTrg>(), statReporter, singleThread_); //throw FileError, X + delHandlerTrg.removeDirWithCallback(folder.getAbstractPath<sideTrg>(), folder.getRelativePath<sideTrg>(), statReporter, singleThread_); //throw FileError, ThreadStopRequest - //TODO: implement parallel folder deletion + //TODO: implement parallel folder deletion - folder.refSubFiles ().clear(); // - folder.refSubLinks ().clear(); //update FolderPair - folder.refSubFolders().clear(); // - folder.removeObject<sideTrg>(); // - } - break; + folder.refSubFiles ().clear(); // + folder.refSubLinks ().clear(); //update FolderPair + folder.refSubFolders().clear(); // + folder.removeObject<sideTrg>(); // + } + break; case SO_OVERWRITE_LEFT: //possible: e.g. manually-resolved dir-traversal conflict case SO_OVERWRITE_RIGHT: // case SO_COPY_METADATA_TO_LEFT: case SO_COPY_METADATA_TO_RIGHT: - reportInfo(txtUpdatingAttributes_, AFS::getDisplayPath(folder.getAbstractPath<sideTrg>())); //throw ThreadStopRequest + reportItemInfo(txtUpdatingAttributes_, folder.getAbstractPath<sideTrg>()); //throw ThreadStopRequest { AsyncItemStatReporter statReporter(1, 0, acb_); @@ -2212,16 +2249,19 @@ void FolderPairSyncer::synchronizeFolderInt(FolderPair& folder, SyncOperation sy //########################################################################################### //returns current attributes of source file -AFS::FileCopyResult FolderPairSyncer::copyFileWithCallback(const FileDescriptor& sourceDescr, //throw FileError, ThreadStopRequest, X +AFS::FileCopyResult FolderPairSyncer::copyFileWithCallback(const FileDescriptor& sourceDescr, const AbstractPath& targetPath, const std::function<void()>& onDeleteTargetFile /*throw X*/, - AsyncPercentStatReporter& statReporter) /*throw ThreadStopRequest*/ + AsyncItemStatReporter& statReporter /*throw ThreadStopRequest*/, + const std::wstring& statusMsg) //throw FileError, ThreadStopRequest, X { const AbstractPath& sourcePath = sourceDescr.path; const AFS::StreamAttributes sourceAttr{sourceDescr.attr.modTime, sourceDescr.attr.fileSize, sourceDescr.attr.filePrint}; auto copyOperation = [&](const AbstractPath& sourcePathTmp) { + PercentStatReporter percentReporter(statusMsg, sourceDescr.attr.fileSize, statReporter); + //already existing + no onDeleteTargetFile: undefined behavior! (e.g. fail/overwrite/auto-rename) const AFS::FileCopyResult result = parallel::copyFileTransactional(sourcePathTmp, sourceAttr, //throw FileError, ErrorFileLocked, ThreadStopRequest, X targetPath, @@ -2236,18 +2276,19 @@ AFS::FileCopyResult FolderPairSyncer::copyFileWithCallback(const FileDescriptor& }, [&](int64_t bytesDelta) //callback runs *outside* singleThread_ lock! => fine { - statReporter.updateStatus(0, bytesDelta); //throw ThreadStopRequest - interruptionPoint(); //throw ThreadStopRequest => not reliably covered by AsyncPercentStatReporter::updateStatus()! + percentReporter.updateDeltaAndStatus(bytesDelta); //throw ThreadStopRequest + interruptionPoint(); //throw ThreadStopRequest => not reliably covered by PercentStatReporter::updateDeltaAndStatus()! }, singleThread_); //#################### Verification ############################# if (verifyCopiedFiles_) { - ZEN_ON_SCOPE_FAIL(try { parallel::removeFilePlain(targetPath, singleThread_); } - catch (FileError&) {}); //delete target if verification fails + reportItemInfo(txtVerifyingFile_, targetPath); //throw ThreadStopRequest - reportInfo(txtVerifyingFile_, AFS::getDisplayPath(targetPath)); //throw ThreadStopRequest + //delete target if verification fails + ZEN_ON_SCOPE_FAIL(try { parallel::removeFilePlain(targetPath, singleThread_); } + catch (const FileError& e) { statReporter.logMessage(e.toString(), PhaseCallback::MsgType::error); /*throw ThreadStopRequest*/ }); //callback runs *outside* singleThread_ lock! => fine auto verifyCallback = [&](int64_t bytesDelta) { interruptionPoint(); }; //throw ThreadStopRequest @@ -2378,7 +2419,7 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime const std::vector<FolderPairSyncCfg>& syncConfig, FolderComparison& folderCmp, WarningDialogs& warnings, - ProcessCallback& callback) + ProcessCallback& callback /*throw X*/) //throw X { //PERF_START; @@ -2424,11 +2465,10 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime } catch (const FileError& e) //failure is not critical => log only { - callback.logInfo(e.toString()); //throw X + callback.logMessage(e.toString(), PhaseCallback::MsgType::warning); //throw X } //-------------------execute basic checks all at once BEFORE starting sync-------------------------------------- - std::vector<unsigned char /*we really want bool*/> skipFolderPair(folderCmp.size(), false); //folder pairs may be skipped after fatal errors were found std::vector<std::tuple<const BaseFolderPair*, int /*conflict count*/, std::vector<SyncStatistics::ConflictInfo>>> checkUnresolvedConflicts; @@ -2439,9 +2479,6 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime std::vector<std::pair<AbstractPath, std::pair<int64_t, int64_t>>> checkDiskSpaceMissing; //base folder / space required / space available - //status of base directories which are set to DeletionPolicy::recycler (and contain actual items to be deleted) - std::map<AbstractPath, bool> recyclerSupported; //expensive to determine on Win XP => buffer + check recycle bin existence only once per base folder! - std::set<AbstractPath> checkVersioningPaths; std::vector<std::pair<AbstractPath, const PathFilter*>> checkVersioningBasePaths; //hard filter creates new logical hierarchies for otherwise equal AbstractPath... @@ -2461,7 +2498,7 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime checkUnresolvedConflicts.emplace_back(&baseFolder, folderPairStat.conflictCount(), folderPairStat.getConflictsPreview()); //consider *all* paths that might be used during versioning limit at some time - if (folderPairCfg.handleDeletion == DeletionPolicy::versioning && + if (folderPairCfg.handleDeletion == DeletionVariant::versioning && folderPairCfg.versioningStyle != VersioningStyle::replace) if (folderPairCfg.versionMaxAgeDays > 0 || folderPairCfg.versionCountMax > 0) //same check as in applyVersioningLimit() checkVersioningLimitPaths.insert(versioningFolderPath); @@ -2535,7 +2572,7 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime continue; } - if (folderPairCfg.handleDeletion == DeletionPolicy::versioning) + if (folderPairCfg.handleDeletion == DeletionVariant::versioning) { //check if user-defined directory for deletion was specified if (AFS::isNullPath(versioningFolderPath)) @@ -2580,39 +2617,12 @@ void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime } catch (const FileError& e) //not critical => log only { - callback.logInfo(e.toString()); //throw X + callback.logMessage(e.toString(), PhaseCallback::MsgType::warning); //throw X } }; const std::pair<int64_t, int64_t> spaceNeeded = MinimumDiskSpaceNeeded::calculate(baseFolder); checkSpace(baseFolder.getAbstractPath<SelectSide::left >(), spaceNeeded.first); checkSpace(baseFolder.getAbstractPath<SelectSide::right>(), spaceNeeded.second); - - //Windows: check if recycle bin really exists; if not, Windows will silently delete, which is just wrong - if (folderPairCfg.handleDeletion == DeletionPolicy::recycler) - { - auto checkRecycler = [&](const AbstractPath& baseFolderPath) - { - assert(!AFS::isNullPath(baseFolderPath)); - if (!AFS::isNullPath(baseFolderPath)) - if (!recyclerSupported.contains(baseFolderPath)) //perf: avoid duplicate checks! - { - callback.updateStatus(replaceCpy(_("Checking recycle bin availability for folder %x..."), L"%x", //throw X - fmtPath(AFS::getDisplayPath(baseFolderPath)))); - bool recSupported = false; - tryReportingError([&] - { - recSupported = AFS::supportsRecycleBin(baseFolderPath); //throw FileError - }, callback); //throw X - - recyclerSupported.emplace(baseFolderPath, recSupported); - } - }; - if (folderPairStat.expectPhysicalDeletion<SelectSide::left>()) - checkRecycler(baseFolder.getAbstractPath<SelectSide::left>()); - - if (folderPairStat.expectPhysicalDeletion<SelectSide::right>()) - checkRecycler(baseFolder.getAbstractPath<SelectSide::right>()); - } } //-------------------------------------------------------------------------------------- @@ -2661,7 +2671,7 @@ break2: ++itPrevi; } - callback.reportWarning(msg, warnings.warnUnresolvedConflicts); + callback.reportWarning(msg, warnings.warnUnresolvedConflicts); //throw X } //check if user accidentally selected wrong directories for sync @@ -2674,7 +2684,7 @@ break2: AFS::getDisplayPath(folderPathL) + L" <-> " + L'\n' + AFS::getDisplayPath(folderPathR); - callback.reportWarning(msg, warnings.warnSignificantDifference); + callback.reportWarning(msg, warnings.warnSignificantDifference); //throw X } //check for sufficient free diskspace @@ -2687,19 +2697,7 @@ break2: TAB_SPACE + _("Required:") + L' ' + formatFilesizeShort(space.first) + L'\n' + TAB_SPACE + _("Available:") + L' ' + formatFilesizeShort(space.second); - callback.reportWarning(msg, warnings.warnNotEnoughDiskSpace); - } - - //windows: check if recycle bin really exists; if not, Windows will silently delete, which is wrong - { - std::wstring msg; - for (const auto& [folderPath, supported] : recyclerSupported) - if (!supported) - msg += L'\n' + AFS::getDisplayPath(folderPath); - - if (!msg.empty()) - callback.reportWarning(_("The recycle bin is not supported by the following folders. Deleted or overwritten files will not be able to be restored:") + L'\n' + msg, - warnings.warnRecyclerMissing); + callback.reportWarning(msg, warnings.warnNotEnoughDiskSpace); //throw X } //check if folders are used by multiple pairs in read/write access @@ -2740,7 +2738,7 @@ break2: else trim(msg); - callback.reportWarning(msg, warnings.warnDependentBaseFolders); + callback.reportWarning(msg, warnings.warnDependentBaseFolders); //throw X } } @@ -2774,15 +2772,15 @@ break2: if (!msg.empty()) callback.reportWarning(_("The versioning folder is contained in a base folder.") + (shouldExclude ? L'\n' + _("The folder should be excluded from synchronization via filter.") : L"") + - msg, warnings.warnVersioningFolderPartOfSync); + msg, warnings.warnVersioningFolderPartOfSync); //throw X } //warn if versioning folder paths differ only in case => possible pessimization for applyVersioningLimit() { std::map<std::pair<AfsDevice, ZstringNoCase>, std::set<AbstractPath>> ciPathAliases; - for (const AbstractPath& ap : checkVersioningLimitPaths) - ciPathAliases[std::pair(ap.afsDevice, ap.afsPath.value)].insert(ap); + for (const AbstractPath& folderPath : checkVersioningLimitPaths) + ciPathAliases[std::pair(folderPath.afsDevice, folderPath.afsPath.value)].insert(folderPath); if (std::any_of(ciPathAliases.begin(), ciPathAliases.end(), [](const auto& item) { return item.second/*aliases*/.size() > 1; })) { @@ -2796,47 +2794,14 @@ break2: } callback.reportWarning(msg, warnings.warnFoldersDifferInCase); //throw X } - //what about /folder and /Folder/subfolder? => yes, inconsistent, but doesn't matter for FFS + //what about /folder and /Folder/subfolder? => yes, inconsistent, but doesn't matter for FFS } //-------------------end of basic checks------------------------------------------ - std::vector<FileError> errorsModTime; //show all warnings as a single message - std::set<VersioningLimitFolder> versionLimitFolders; - //------------------- show warnings after synchronization -------------------------------------- - //report errors when setting modification time as (a single) warning only! - const int exeptionCount = std::uncaught_exceptions(); - ZEN_ON_SCOPE_EXIT - ( - //*INDENT-OFF* - if (!errorsModTime.empty()) - { - size_t previewCount = 0; - std::wstring msg; - for (const FileError& e : errorsModTime) - { - const std::wstring& singleMsg = replaceCpy(e.toString(), L"\n\n", L'\n'); - msg += singleMsg + L"\n\n"; - - if (++previewCount >= MODTIME_ERRORS_PREVIEW_MAX) - break; - } - msg.resize(msg.size() - 2); - - if (errorsModTime.size() > previewCount) - msg += L"\n [...] " + replaceCpy(_P("Showing %y of 1 item", "Showing %y of %x items", errorsModTime.size()), //%x used as plural form placeholder! - L"%y", formatNumber(previewCount)); + bool recyclerMissingReportOnce = false; //prompt user only *once* per sync, not per failed item! - const bool scopeFail = std::uncaught_exceptions() > exeptionCount; - if (!scopeFail) - callback.reportWarning(msg, warnings.warnModificationTimeError); //throw X - else //at least log warnings when sync is cancelled - try { callback.logInfo(msg); /*throw X*/} catch (...) {}; - } - //*INDENT-ON* - ); - //---------------------------------------------------------------------------------------------- class PcbNoThrow : public PhaseCallback { public: @@ -2848,11 +2813,11 @@ break2: void requestUiUpdate(bool force) override { try { cb_.requestUiUpdate(force); /*throw X*/} catch (...) {}; } void updateStatus(std::wstring&& msg) override { try { cb_.updateStatus(std::move(msg)); /*throw X*/} catch (...) {}; } - void logInfo(const std::wstring& msg) override { try { cb_.logInfo(msg); /*throw X*/} catch (...) {}; } + void logMessage(const std::wstring& msg, MsgType type) override { try { cb_.logMessage(msg, type); /*throw X*/} catch (...) {}; } - void reportWarning(const std::wstring& msg, bool& warningActive) override { logInfo(msg); /*ignore*/ } - Response reportError (const ErrorInfo& errorInfo) override { logInfo(errorInfo.msg); return Response::ignore; } - void reportFatalError(const std::wstring& msg) override { logInfo(msg); /*ignore*/ } + void reportWarning(const std::wstring& msg, bool& warningActive) override { logMessage(msg, MsgType::warning); /*ignore*/ } + Response reportError (const ErrorInfo& errorInfo) override { logMessage(errorInfo.msg, MsgType::error); return Response::ignore; } + void reportFatalError(const std::wstring& msg) override { logMessage(msg, MsgType::error); /*ignore*/ } private: ProcessCallback& cb_; @@ -2871,9 +2836,9 @@ break2: continue; //------------------------------------------------------------------------------------------ - callback.logInfo(_("Synchronizing folder pair:") + L' ' + getVariantNameWithSymbol(folderPairCfg.syncVar) + L'\n' + //throw X - TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::left >()) + L'\n' + - TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::right>())); + callback.logMessage(_("Synchronizing folder pair:") + L' ' + getVariantNameWithSymbol(folderPairCfg.syncVar) + L'\n' + //throw X + TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::left >()) + L'\n' + + TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::right>()), PhaseCallback::MsgType::info); //------------------------------------------------------------------------------------------ //checking a second time: 1. a long time may have passed since syncing the previous folder pairs! @@ -2912,28 +2877,20 @@ break2: baseFolder.getAbstractPath<SelectSide::right>()); //throw FileError }, callback); //throw X - - auto getEffectiveDeletionPolicy = [&](const AbstractPath& baseFolderPath) -> DeletionPolicy - { - if (folderPairCfg.handleDeletion == DeletionPolicy::recycler) - { - auto it = recyclerSupported.find(baseFolderPath); - if (it != recyclerSupported.end()) //buffer filled during intro checks (but only if deletions are expected) - if (!it->second) - return DeletionPolicy::permanent; //Windows' ::SHFileOperation() will do this anyway, but we have a better and faster deletion routine (e.g. on networks) - } - return folderPairCfg.handleDeletion; - }; const AbstractPath versioningFolderPath = createAbstractPath(folderPairCfg.versioningFolderPhrase); DeletionHandler delHandlerL(baseFolder.getAbstractPath<SelectSide::left>(), - getEffectiveDeletionPolicy(baseFolder.getAbstractPath<SelectSide::left>()), + recyclerMissingReportOnce, + warnings.warnRecyclerMissing, + folderPairCfg.handleDeletion, versioningFolderPath, folderPairCfg.versioningStyle, std::chrono::system_clock::to_time_t(syncStartTime)); DeletionHandler delHandlerR(baseFolder.getAbstractPath<SelectSide::right>(), - getEffectiveDeletionPolicy(baseFolder.getAbstractPath<SelectSide::right>()), + recyclerMissingReportOnce, + warnings.warnRecyclerMissing, + folderPairCfg.handleDeletion, versioningFolderPath, folderPairCfg.versioningStyle, std::chrono::system_clock::to_time_t(syncStartTime)); @@ -2949,7 +2906,6 @@ break2: FolderPairSyncer::SyncCtx syncCtx = { verifyCopiedFiles, copyPermissionsFp, failSafeFileCopy, - errorsModTime, delHandlerL, delHandlerR, }; FolderPairSyncer::runSync(syncCtx, baseFolder, callback); @@ -2959,7 +2915,7 @@ break2: delHandlerR.tryCleanup(callback); // guardDelCleanup.dismiss(); - if (folderPairCfg.handleDeletion == DeletionPolicy::versioning && + if (folderPairCfg.handleDeletion == DeletionVariant::versioning && folderPairCfg.versioningStyle != VersioningStyle::replace) versionLimitFolders.insert( { diff --git a/FreeFileSync/Source/base/synchronization.h b/FreeFileSync/Source/base/synchronization.h index 6a4f4b0d..7294e642 100644 --- a/FreeFileSync/Source/base/synchronization.h +++ b/FreeFileSync/Source/base/synchronization.h @@ -35,9 +35,6 @@ public: int deleteCount() const { return selectParam<side>(deleteLeft_, deleteRight_); } int deleteCount() const { return deleteLeft_ + deleteRight_; } - template <SelectSide side> - bool expectPhysicalDeletion() const { return selectParam<side>(physicalDeleteLeft_, physicalDeleteRight_); } - int64_t getBytesToProcess() const { return bytesToProcess_; } size_t rowCount () const { return rowsTotal_; } @@ -62,8 +59,6 @@ private: int updateRight_ = 0; int deleteLeft_ = 0; int deleteRight_ = 0; - bool physicalDeleteLeft_ = false; //at least 1 item will be deleted; considers most "update" cases which also delete items - bool physicalDeleteRight_ = false; // int64_t bytesToProcess_ = 0; size_t rowsTotal_ = 0; @@ -78,7 +73,7 @@ struct FolderPairSyncCfg { SyncVariant syncVar; bool saveSyncDB; //save database if in automatic mode or dection of moved files is active - DeletionPolicy handleDeletion; + DeletionVariant handleDeletion; Zstring versioningFolderPhrase; //unresolved directory names as entered by user! VersioningStyle versioningStyle; int versionMaxAgeDays; @@ -98,7 +93,7 @@ void synchronize(const std::chrono::system_clock::time_point& syncStartTime, const std::vector<FolderPairSyncCfg>& syncConfig, //CONTRACT: syncConfig and folderCmp correspond row-wise! FolderComparison& folderCmp, // WarningDialogs& warnings, - ProcessCallback& callback); + ProcessCallback& callback /*throw X*/); //throw X } #endif //SYNCHRONIZATION_H_8913470815943295 diff --git a/FreeFileSync/Source/base/versioning.cpp b/FreeFileSync/Source/base/versioning.cpp index 1a17c6b2..0a82c153 100644 --- a/FreeFileSync/Source/base/versioning.cpp +++ b/FreeFileSync/Source/base/versioning.cpp @@ -564,7 +564,7 @@ void fff::applyVersioningLimit(const std::set<VersioningLimitFolder>& folderLimi { const std::wstring errMsg = tryReportingError([&] //throw ThreadStopRequest { - ctx.acb.reportInfo(txtRemoving + AFS::getDisplayPath(ctx.itemPath)); //throw ThreadStopRequest + reportInfo(txtRemoving + AFS::getDisplayPath(ctx.itemPath), ctx.acb); //throw ThreadStopRequest if (isSymlink) AFS::removeSymlinkIfExists(ctx.itemPath); //throw FileError else diff --git a/FreeFileSync/Source/base_tools.cpp b/FreeFileSync/Source/base_tools.cpp index e642463a..ee7de8cd 100644 --- a/FreeFileSync/Source/base_tools.cpp +++ b/FreeFileSync/Source/base_tools.cpp @@ -59,28 +59,28 @@ void fff::logNonDefaultSettings(const XmlGlobalSettings& activeSettings, PhaseCa std::wstring changedSettingsMsg; if (activeSettings.failSafeFileCopy != defaultSettings.failSafeFileCopy) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Fail-safe file copy")) + L" - " + (activeSettings.failSafeFileCopy ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Fail-safe file copy")) + L": " + (activeSettings.failSafeFileCopy ? _("Enabled") : _("Disabled")); if (activeSettings.copyLockedFiles != defaultSettings.copyLockedFiles) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Copy locked files")) + L" - " + (activeSettings.copyLockedFiles ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Copy locked files")) + L": " + (activeSettings.copyLockedFiles ? _("Enabled") : _("Disabled")); if (activeSettings.copyFilePermissions != defaultSettings.copyFilePermissions) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Copy file access permissions")) + L" - " + (activeSettings.copyFilePermissions ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Copy file access permissions")) + L": " + (activeSettings.copyFilePermissions ? _("Enabled") : _("Disabled")); if (activeSettings.fileTimeTolerance != defaultSettings.fileTimeTolerance) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("File time tolerance")) + L" - " + numberTo<std::wstring>(activeSettings.fileTimeTolerance); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("File time tolerance")) + L": " + formatNumber(activeSettings.fileTimeTolerance); if (activeSettings.runWithBackgroundPriority != defaultSettings.runWithBackgroundPriority) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Run with background priority")) + L" - " + (activeSettings.runWithBackgroundPriority ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Run with background priority")) + L": " + (activeSettings.runWithBackgroundPriority ? _("Enabled") : _("Disabled")); if (activeSettings.createLockFile != defaultSettings.createLockFile) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Lock directories during sync")) + L" - " + (activeSettings.createLockFile ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Lock directories during sync")) + L": " + (activeSettings.createLockFile ? _("Enabled") : _("Disabled")); if (activeSettings.verifyFileCopy != defaultSettings.verifyFileCopy) - changedSettingsMsg += L"\n" + (TAB_SPACE + _("Verify copied files")) + L" - " + (activeSettings.verifyFileCopy ? _("Enabled") : _("Disabled")); + changedSettingsMsg += L"\n" + (TAB_SPACE + _("Verify copied files")) + L": " + (activeSettings.verifyFileCopy ? _("Enabled") : _("Disabled")); if (!changedSettingsMsg.empty()) - callback.logInfo(_("Using non-default global settings:") + changedSettingsMsg); //throw X + callback.logMessage(_("Using non-default global settings:") + changedSettingsMsg, PhaseCallback::MsgType::info); //throw X } diff --git a/FreeFileSync/Source/config.cpp b/FreeFileSync/Source/config.cpp index 21bb4e0c..04e42abf 100644 --- a/FreeFileSync/Source/config.cpp +++ b/FreeFileSync/Source/config.cpp @@ -349,32 +349,32 @@ bool readText(const std::string& input, GridIconSize& value) template <> inline -void writeText(const DeletionPolicy& value, std::string& output) +void writeText(const DeletionVariant& value, std::string& output) { switch (value) { - case DeletionPolicy::permanent: + case DeletionVariant::permanent: output = "Permanent"; break; - case DeletionPolicy::recycler: + case DeletionVariant::recycler: output = "RecycleBin"; break; - case DeletionPolicy::versioning: + case DeletionVariant::versioning: output = "Versioning"; break; } } template <> inline -bool readText(const std::string& input, DeletionPolicy& value) +bool readText(const std::string& input, DeletionVariant& value) { const std::string tmp = trimCpy(input); if (tmp == "Permanent") - value = DeletionPolicy::permanent; + value = DeletionVariant::permanent; else if (tmp == "RecycleBin") - value = DeletionPolicy::recycler; + value = DeletionVariant::recycler; else if (tmp == "Versioning") - value = DeletionPolicy::versioning; + value = DeletionVariant::versioning; else return false; return true; @@ -970,10 +970,11 @@ void writeStruc(const ConfigFileItem& value, XmlElement& output) if (value.backColor.IsOk()) { - const auto& [highR, lowR] = hexify(value.backColor.Red ()); - const auto& [highG, lowG] = hexify(value.backColor.Green()); - const auto& [highB, lowB] = hexify(value.backColor.Blue ()); - output.setAttribute("Color", std::string({highR, lowR, highG, lowG, highB, lowB})); + assert(value.backColor.Alpha() == 255); + const auto& [rh, rl] = hexify(value.backColor.Red ()); + const auto& [gh, gl] = hexify(value.backColor.Green()); + const auto& [bh, bl] = hexify(value.backColor.Blue ()); + output.setAttribute("Color", std::string({rh, rl, gh, gl, bh, bl})); } } @@ -1033,7 +1034,7 @@ void readConfig(const XmlIn& in, SyncConfig& syncCfg, std::map<AfsDevice, size_t { readConfig(in, syncCfg.directionCfg); - in["DeletionPolicy" ](syncCfg.handleDeletion); + in["DeletionPolicy" ](syncCfg.deletionVariant); in["VersioningFolder"](syncCfg.versioningFolderPhrase); if (formatVer < 12) //TODO: remove if parameter migration after some time! 2018-06-21 @@ -1517,7 +1518,6 @@ void readConfig(const XmlIn& in, XmlGlobalSettings& cfg, int formatVer) inOpt["WarnSignificantDifference" ].attribute("Enabled", cfg.warnDlgs.warnSignificantDifference); inOpt["WarnRecycleBinNotAvailable" ].attribute("Enabled", cfg.warnDlgs.warnRecyclerMissing); inOpt["WarnInputFieldEmpty" ].attribute("Enabled", cfg.warnDlgs.warnInputFieldEmpty); - inOpt["WarnModificationTimeError" ].attribute("Enabled", cfg.warnDlgs.warnModificationTimeError); inOpt["WarnDependentFolderPair" ].attribute("Enabled", cfg.warnDlgs.warnDependentFolderPair); inOpt["WarnDependentBaseFolders" ].attribute("Enabled", cfg.warnDlgs.warnDependentBaseFolders); inOpt["WarnDirectoryLockFailed" ].attribute("Enabled", cfg.warnDlgs.warnDirectoryLockFailed); @@ -1540,7 +1540,6 @@ void readConfig(const XmlIn& in, XmlGlobalSettings& cfg, int formatVer) inOpt["WarnSignificantDifference" ].attribute("Show", cfg.warnDlgs.warnSignificantDifference); inOpt["WarnRecycleBinNotAvailable" ].attribute("Show", cfg.warnDlgs.warnRecyclerMissing); inOpt["WarnInputFieldEmpty" ].attribute("Show", cfg.warnDlgs.warnInputFieldEmpty); - inOpt["WarnModificationTimeError" ].attribute("Show", cfg.warnDlgs.warnModificationTimeError); inOpt["WarnDependentFolderPair" ].attribute("Show", cfg.warnDlgs.warnDependentFolderPair); inOpt["WarnDependentBaseFolders" ].attribute("Show", cfg.warnDlgs.warnDependentBaseFolders); inOpt["WarnDirectoryLockFailed" ].attribute("Show", cfg.warnDlgs.warnDirectoryLockFailed); @@ -2049,6 +2048,7 @@ std::pair<ConfigType, std::wstring /*warningMsg*/> parseConfig(const XmlDoc& doc if (formatVer < currentXmlFormatVer) try { fff::writeConfig(cfg, filePath); /*throw FileError*/ } catch (FileError&) { assert(false); } //don't bother user! + warn_static("at least log on failure!") } catch (const FileError& e) { warningMsg = e.toString(); } @@ -2131,7 +2131,8 @@ std::pair<XmlGuiConfig, std::wstring /*warningMsg*/> fff::readAnyConfig(const st warningMsgAll += warningMsg + L"\n\n"; } else - throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath))); + throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath)), + _("Unexpected file extension:") + L' ' + fmtPath(getFileExtension(filePath))); } cfg.mainCfg = merge(mainCfgs); @@ -2173,7 +2174,7 @@ void writeConfig(const SyncConfig& syncCfg, const std::map<AfsDevice, size_t>& d { writeConfig(syncCfg.directionCfg, out); - out["DeletionPolicy" ](syncCfg.handleDeletion); + out["DeletionPolicy" ](syncCfg.deletionVariant); out["VersioningFolder"](syncCfg.versioningFolderPhrase); const size_t parallelOps = getDeviceParallelOps(deviceParallelOps, syncCfg.versioningFolderPhrase); @@ -2340,7 +2341,6 @@ void writeConfig(const XmlGlobalSettings& cfg, XmlOut& out) outOpt["WarnSignificantDifference" ].attribute("Show", cfg.warnDlgs.warnSignificantDifference); outOpt["WarnRecycleBinNotAvailable" ].attribute("Show", cfg.warnDlgs.warnRecyclerMissing); outOpt["WarnInputFieldEmpty" ].attribute("Show", cfg.warnDlgs.warnInputFieldEmpty); - outOpt["WarnModificationTimeError" ].attribute("Show", cfg.warnDlgs.warnModificationTimeError); outOpt["WarnDependentFolderPair" ].attribute("Show", cfg.warnDlgs.warnDependentFolderPair); outOpt["WarnDependentBaseFolders" ].attribute("Show", cfg.warnDlgs.warnDependentBaseFolders); outOpt["WarnDirectoryLockFailed" ].attribute("Show", cfg.warnDlgs.warnDirectoryLockFailed); diff --git a/FreeFileSync/Source/ffs_paths.cpp b/FreeFileSync/Source/ffs_paths.cpp index 84d8f74c..ec17c609 100644 --- a/FreeFileSync/Source/ffs_paths.cpp +++ b/FreeFileSync/Source/ffs_paths.cpp @@ -81,7 +81,8 @@ Zstring fff::getConfigDirPath() catch (const FileError& e) { assert(false); - std::cerr << utfTo<std::string>(e.toString()) << '\n'; + std::cerr << utfTo<std::string>(e.toString()) + '\n'; + warn_static("at least log on failure!") } return configPath; }(); diff --git a/FreeFileSync/Source/localization.cpp b/FreeFileSync/Source/localization.cpp index d0aa5692..adbd1677 100644 --- a/FreeFileSync/Source/localization.cpp +++ b/FreeFileSync/Source/localization.cpp @@ -169,6 +169,7 @@ std::vector<TranslationInfo> loadTranslations(const Zstring& zipPath) //throw Fi }); } catch (lng::ParsingError&) { assert(false); } + warn_static("at least log on failure!") std::sort(translations.begin(), translations.end(), [](const TranslationInfo& lhs, const TranslationInfo& rhs) { @@ -301,7 +302,7 @@ public: private: const wxString canonicalName_; - MemoryStreamOut<std::string> moBuf_; + MemoryStreamOut moBuf_; }; diff --git a/FreeFileSync/Source/log_file.cpp b/FreeFileSync/Source/log_file.cpp index 71255ba5..410c78d9 100644 --- a/FreeFileSync/Source/log_file.cpp +++ b/FreeFileSync/Source/log_file.cpp @@ -369,36 +369,29 @@ std::string generateLogFooterHtml(const std::wstring& logFilePath /*optional*/, //-> Astyle fucks up! => no INDENT-ON -void streamToLogFile(const ProcessSummary& summary, //throw FileError + //write log items in blocks instead of creating one big string: memory allocation might fail; think 1 million entries! +template <class Function> +void streamToLogFile(const ProcessSummary& summary, const ErrorLog& log, - AFS::OutputStream& streamOut, - LogFileFormat logFormat) + Function stringOut /*(const std::string& s); throw X*/, + LogFileFormat logFormat) //throw FileError, X { const int logItemsTotal = log.end() - log.begin(); const int logPreviewItemsMax = std::numeric_limits<int>::max(); - std::string buffer = logFormat == LogFileFormat::html ? + stringOut(logFormat == LogFileFormat::html ? generateLogHeaderHtml(summary, log, LOG_PREVIEW_FAIL_MAX) : - generateLogHeaderTxt (summary, log, LOG_PREVIEW_FAIL_MAX); + generateLogHeaderTxt (summary, log, LOG_PREVIEW_FAIL_MAX)); //throw X - //write log items in blocks instead of creating one big string: memory allocation might fail; think 1 million entries! for (const LogEntry& entry : log) - { - buffer += logFormat == LogFileFormat::html ? + stringOut(logFormat == LogFileFormat::html ? formatMessageHtml(entry) : - formatMessage (entry); + formatMessage (entry)); //throw X - streamOut.write(&buffer[0], buffer.size()); //throw FileError, X - buffer.clear(); - } - - buffer += logFormat == LogFileFormat::html ? - generateLogFooterHtml(std::wstring() /*logFilePath*/, logItemsTotal, logPreviewItemsMax) : //throw FileError - generateLogFooterTxt (std::wstring() /*logFilePath*/, logItemsTotal, logPreviewItemsMax); //throw FileError + stringOut(logFormat == LogFileFormat::html ? + generateLogFooterHtml(std::wstring() /*logFilePath*/, logItemsTotal, logPreviewItemsMax) /*throw FileError*/: + generateLogFooterTxt (std::wstring() /*logFilePath*/, logItemsTotal, logPreviewItemsMax) /*throw FileError*/); //throw FileError, X //=> log file path is irrelevant, except when sending email! - - //don't forget to flush: - streamOut.write(&buffer[0], buffer.size()); //throw FileError, X } @@ -430,9 +423,21 @@ void saveNewLogFile(const AbstractPath& logFilePath, //throw FileError, X }; //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) - std::unique_ptr<AFS::OutputStream> logFileStream = AFS::getOutputStream(logFilePath, std::nullopt /*streamSize*/, std::nullopt /*modTime*/, notifyUnbufferedIO); //throw FileError - streamToLogFile(summary, log, *logFileStream, logFormat); //throw FileError, X - logFileStream->finalize(); //throw FileError, X + std::unique_ptr<AFS::OutputStream> logFileOut = AFS::getOutputStream(logFilePath, + std::nullopt /*streamSize*/, + std::nullopt /*modTime*/); //throw FileError + + BufferedOutputStream streamOut([&](const void* buffer, size_t bytesToWrite) + { + return logFileOut->tryWrite(buffer, bytesToWrite, notifyUnbufferedIO); //throw FileError, X + }, + logFileOut->getBlockSize()); + + auto stringOut = [&](const std::string& str){ streamOut.write(str.data(), str.size()); }; //throw FileError, X + streamToLogFile(summary, log, stringOut, logFormat); //throw FileError, X + streamOut.flushBuffer(); //throw FileError, X + + logFileOut->finalize(notifyUnbufferedIO); //throw FileError, X } diff --git a/FreeFileSync/Source/ui/batch_status_handler.cpp b/FreeFileSync/Source/ui/batch_status_handler.cpp index f4336548..db4eb8ad 100644 --- a/FreeFileSync/Source/ui/batch_status_handler.cpp +++ b/FreeFileSync/Source/ui/batch_status_handler.cpp @@ -140,8 +140,8 @@ BatchStatusHandler::Result BatchStatusHandler::reportResults(const Zstring& post syncResult == SyncResult::finishedError))) try { - sendLogAsEmail(notifyEmail, summary, errorLog_, logFilePath, notifyStatusNoThrow); //throw FileError logMsg(errorLog_, replaceCpy(_("Sending email notification to %x"), L"%x", utfTo<std::wstring>(notifyEmail)), MSG_TYPE_INFO); + sendLogAsEmail(notifyEmail, summary, errorLog_, logFilePath, notifyStatusNoThrow); //throw FileError } catch (const FileError& e) { logMsg(errorLog_, e.toString(), MSG_TYPE_ERROR); } @@ -271,9 +271,21 @@ void BatchStatusHandler::updateDataProcessed(int itemsDelta, int64_t bytesDelta) } -void BatchStatusHandler::logInfo(const std::wstring& msg) +void BatchStatusHandler::logMessage(const std::wstring& msg, MsgType type) { - logMsg(errorLog_, msg, MSG_TYPE_INFO); + logMsg(errorLog_, msg, [&] + { + switch (type) + { + //*INDENT-OFF* + case MsgType::info: return MSG_TYPE_INFO; + case MsgType::warning: return MSG_TYPE_WARNING; + case MsgType::error: return MSG_TYPE_ERROR; + //*INDENT-ON* + } + assert(false); + return MSG_TYPE_ERROR; + }()); requestUiUpdate(false /*force*/); //throw AbortProcess } diff --git a/FreeFileSync/Source/ui/batch_status_handler.h b/FreeFileSync/Source/ui/batch_status_handler.h index 62bd1a7d..db35190f 100644 --- a/FreeFileSync/Source/ui/batch_status_handler.h +++ b/FreeFileSync/Source/ui/batch_status_handler.h @@ -36,7 +36,7 @@ public: ~BatchStatusHandler(); void initNewPhase (int itemsTotal, int64_t bytesTotal, ProcessPhase phaseID) override; // - void logInfo (const std::wstring& msg) override; // + void logMessage (const std::wstring& msg, MsgType type) override; // void reportWarning (const std::wstring& msg, bool& warningActive) override; //throw AbortProcess Response reportError (const ErrorInfo& errorInfo) override; // void reportFatalError(const std::wstring& msg) override; // diff --git a/FreeFileSync/Source/ui/cfg_grid.cpp b/FreeFileSync/Source/ui/cfg_grid.cpp index 8161d13e..00d5d33c 100644 --- a/FreeFileSync/Source/ui/cfg_grid.cpp +++ b/FreeFileSync/Source/ui/cfg_grid.cpp @@ -149,17 +149,23 @@ void ConfigView::setLastRunStats(const std::vector<Zstring>& filePaths, const La } -void ConfigView::setBackColor(const std::vector<Zstring>& filePaths, const wxColor& col) +void ConfigView::setBackColor(const std::vector<Zstring>& filePaths, const wxColor& col, bool previewOnly) { for (const Zstring& filePath : filePaths) - { - auto it = cfgList_.find(filePath); - assert(it != cfgList_.end()); - if (it != cfgList_.end()) - it->second.cfgItem.backColor = col; - } + if (auto it = cfgList_.find(filePath); + it != cfgList_.end()) + { + if (previewOnly) + it->second.cfgItem.backColorPreview = col; + else + { + it->second.cfgItem.backColor = col; + it->second.cfgItem.backColorPreview = wxNullColour; + } + } + else assert(false); - if (sortColumn_ == ColumnTypeCfg::name) + if (!previewOnly && sortColumn_ == ColumnTypeCfg::name) sortListView(); //needed if top element of colored-group is removed } @@ -363,27 +369,31 @@ private: { case ColumnTypeCfg::name: { - if (item->cfgItem.backColor.IsOk()) + wxColor backColor = item->cfgItem.backColor; + if (item->cfgItem.backColorPreview.IsOk()) + backColor = item->cfgItem.backColorPreview; + + if (backColor.IsOk()) { wxRect rectTmp2 = rectTmp; - if (!selected) + if (!selected || item->cfgItem.backColorPreview.IsOk()) { rectTmp2.width = rectTmp.width * 2 / 3; - clearArea(dc, rectTmp2, item->cfgItem.backColor); //accessibility: always set both foreground AND background colors! - textColor.Set(*wxBLACK); // + clearArea(dc, rectTmp2, backColor); //accessibility: always set both foreground AND background colors! + textColor.Set(*wxBLACK); // rectTmp2.x += rectTmp2.width; rectTmp2.width = rectTmp.width - rectTmp2.width; - dc.GradientFillLinear(rectTmp2, item->cfgItem.backColor, wxSystemSettings::GetColour(wxSYS_COLOUR_WINDOW), wxEAST); + dc.GradientFillLinear(rectTmp2, backColor, wxSystemSettings::GetColour(wxSYS_COLOUR_WINDOW), wxEAST); } else //always show a glimpse of the background color { rectTmp2.width = getColumnGapLeft() + getDefaultMenuIconSize(); - clearArea(dc, rectTmp2, item->cfgItem.backColor); + clearArea(dc, rectTmp2, backColor); rectTmp2.x += rectTmp2.width; rectTmp2.width = getColumnGapLeft(); - dc.GradientFillLinear(rectTmp2, item->cfgItem.backColor, wxSystemSettings::GetColour(wxSYS_COLOUR_HIGHLIGHT), wxEAST); + dc.GradientFillLinear(rectTmp2, backColor, wxSystemSettings::GetColour(wxSYS_COLOUR_HIGHLIGHT), wxEAST); } } diff --git a/FreeFileSync/Source/ui/cfg_grid.h b/FreeFileSync/Source/ui/cfg_grid.h index ce1f329a..bca0f267 100644 --- a/FreeFileSync/Source/ui/cfg_grid.h +++ b/FreeFileSync/Source/ui/cfg_grid.h @@ -35,6 +35,7 @@ struct ConfigFileItem AbstractPath logFilePath = getNullPath(); //ANY last sync attempt (including aborted syncs) SyncResult logResult = SyncResult::aborted; // wxColor backColor; + wxColor backColorPreview; //while the folder picker is shown }; @@ -104,7 +105,7 @@ public: AbstractPath logFilePath; //optional }; void setLastRunStats(const std::vector<Zstring>& filePaths, const LastRunStats& lastRun); - void setBackColor(const std::vector<Zstring>& filePaths, const wxColor& col); + void setBackColor(const std::vector<Zstring>& filePaths, const wxColor& col, bool previewOnly = false); struct Details { diff --git a/FreeFileSync/Source/ui/file_grid.cpp b/FreeFileSync/Source/ui/file_grid.cpp index 824648b2..6c8a1f4a 100644 --- a/FreeFileSync/Source/ui/file_grid.cpp +++ b/FreeFileSync/Source/ui/file_grid.cpp @@ -2056,7 +2056,7 @@ void IconManager::startIconUpdater() { assert(iconUpdater_); if (iconUpdater_) i void filegrid::setupIcons(Grid& gridLeft, Grid& gridCenter, Grid& gridRight, bool showFileIcons, IconBuffer::IconSize sz) { - auto* provLeft = dynamic_cast<GridDataLeft *>(gridLeft .getDataProvider()); + auto* provLeft = dynamic_cast<GridDataLeft*>(gridLeft .getDataProvider()); auto* provRight = dynamic_cast<GridDataRight*>(gridRight.getDataProvider()); if (provLeft && provRight) diff --git a/FreeFileSync/Source/ui/gui_generated.cpp b/FreeFileSync/Source/ui/gui_generated.cpp index 9ec3baf5..1fb1ac6f 100644 --- a/FreeFileSync/Source/ui/gui_generated.cpp +++ b/FreeFileSync/Source/ui/gui_generated.cpp @@ -3451,7 +3451,7 @@ CompareProgressDlgGenerated::CompareProgressDlgGenerated( wxWindow* parent, wxWi m_panelItemStats->SetBackgroundColour( wxSystemSettings::GetColour( wxSYS_COLOUR_BTNFACE ) ); wxBoxSizer* bSizer291; - bSizer291 = new wxBoxSizer( wxVERTICAL ); + bSizer291 = new wxBoxSizer( wxHORIZONTAL ); ffgSizer111 = new wxFlexGridSizer( 0, 2, 5, 5 ); ffgSizer111->SetFlexibleDirection( wxBOTH ); @@ -3490,19 +3490,19 @@ CompareProgressDlgGenerated::CompareProgressDlgGenerated( wxWindow* parent, wxWi ffgSizer111->Add( m_staticTextBytesRemaining, 0, wxALIGN_RIGHT|wxALIGN_CENTER_VERTICAL, 5 ); - bSizer291->Add( ffgSizer111, 0, wxALL, 5 ); + bSizer291->Add( ffgSizer111, 0, wxALL|wxALIGN_CENTER_VERTICAL, 5 ); m_panelItemStats->SetSizer( bSizer291 ); m_panelItemStats->Layout(); bSizer291->Fit( m_panelItemStats ); - bSizer199->Add( m_panelItemStats, 0, wxALIGN_CENTER_VERTICAL|wxTOP|wxBOTTOM|wxRIGHT, 10 ); + bSizer199->Add( m_panelItemStats, 0, wxTOP|wxBOTTOM|wxRIGHT|wxEXPAND, 10 ); m_panelTimeStats = new wxPanel( this, wxID_ANY, wxDefaultPosition, wxDefaultSize, 0 ); m_panelTimeStats->SetBackgroundColour( wxSystemSettings::GetColour( wxSYS_COLOUR_BTNFACE ) ); wxBoxSizer* bSizer292; - bSizer292 = new wxBoxSizer( wxVERTICAL ); + bSizer292 = new wxBoxSizer( wxHORIZONTAL ); ffgSizer112 = new wxFlexGridSizer( 0, 1, 5, 5 ); ffgSizer112->SetFlexibleDirection( wxBOTH ); @@ -3533,13 +3533,13 @@ CompareProgressDlgGenerated::CompareProgressDlgGenerated( wxWindow* parent, wxWi ffgSizer112->Add( m_staticTextTimeRemaining, 0, wxALIGN_CENTER_VERTICAL|wxALIGN_RIGHT, 5 ); - bSizer292->Add( ffgSizer112, 0, wxALL, 5 ); + bSizer292->Add( ffgSizer112, 0, wxALL|wxALIGN_CENTER_VERTICAL, 5 ); m_panelTimeStats->SetSizer( bSizer292 ); m_panelTimeStats->Layout(); bSizer292->Fit( m_panelTimeStats ); - bSizer199->Add( m_panelTimeStats, 0, wxALIGN_CENTER_VERTICAL|wxTOP|wxBOTTOM|wxRIGHT, 10 ); + bSizer199->Add( m_panelTimeStats, 0, wxTOP|wxBOTTOM|wxRIGHT|wxEXPAND, 10 ); bSizerErrorsRetry = new wxBoxSizer( wxHORIZONTAL ); @@ -5112,7 +5112,7 @@ AboutDlgGenerated::AboutDlgGenerated( wxWindow* parent, wxWindowID id, const wxS bSizer183 = new wxBoxSizer( wxHORIZONTAL ); m_bitmapAnimalSmall = new wxStaticBitmap( m_panelDonate, wxID_ANY, wxNullBitmap, wxDefaultPosition, wxDefaultSize, 0 ); - bSizer183->Add( m_bitmapAnimalSmall, 0, wxALIGN_CENTER_VERTICAL|wxTOP|wxBOTTOM|wxLEFT, 5 ); + bSizer183->Add( m_bitmapAnimalSmall, 0, wxALIGN_CENTER_VERTICAL|wxALL, 5 ); m_panel39 = new wxPanel( m_panelDonate, wxID_ANY, wxDefaultPosition, wxDefaultSize, wxTAB_TRAVERSAL ); m_panel39->SetBackgroundColour( wxColour( 248, 248, 248 ) ); diff --git a/FreeFileSync/Source/ui/gui_status_handler.cpp b/FreeFileSync/Source/ui/gui_status_handler.cpp index 7702cb6d..4c751083 100644 --- a/FreeFileSync/Source/ui/gui_status_handler.cpp +++ b/FreeFileSync/Source/ui/gui_status_handler.cpp @@ -188,9 +188,21 @@ void StatusHandlerTemporaryPanel::initNewPhase(int itemsTotal, int64_t bytesTota } -void StatusHandlerTemporaryPanel::logInfo(const std::wstring& msg) +void StatusHandlerTemporaryPanel::logMessage(const std::wstring& msg, MsgType type) { - logMsg(errorLog_, msg, MSG_TYPE_INFO); + logMsg(errorLog_, msg, [&] + { + switch (type) + { + //*INDENT-OFF* + case MsgType::info: return MSG_TYPE_INFO; + case MsgType::warning: return MSG_TYPE_WARNING; + case MsgType::error: return MSG_TYPE_ERROR; + //*INDENT-ON* + } + assert(false); + return MSG_TYPE_ERROR; + }()); requestUiUpdate(false /*force*/); //throw AbortProcess } @@ -449,8 +461,8 @@ StatusHandlerFloatingDialog::Result StatusHandlerFloatingDialog::reportResults(c syncResult == SyncResult::finishedError))) try { - sendLogAsEmail(notifyEmail, summary, errorLog_, logFilePath, notifyStatusNoThrow); //throw FileError logMsg(errorLog_, replaceCpy(_("Sending email notification to %x"), L"%x", utfTo<std::wstring>(notifyEmail)), MSG_TYPE_INFO); + sendLogAsEmail(notifyEmail, summary, errorLog_, logFilePath, notifyStatusNoThrow); //throw FileError } catch (const FileError& e) { logMsg(errorLog_, e.toString(), MSG_TYPE_ERROR); } @@ -567,9 +579,22 @@ void StatusHandlerFloatingDialog::initNewPhase(int itemsTotal, int64_t bytesTota } -void StatusHandlerFloatingDialog::logInfo(const std::wstring& msg) + +void StatusHandlerFloatingDialog::logMessage(const std::wstring& msg, MsgType type) { - logMsg(errorLog_, msg, MSG_TYPE_INFO); + logMsg(errorLog_, msg, [&] + { + switch (type) + { + //*INDENT-OFF* + case MsgType::info: return MSG_TYPE_INFO; + case MsgType::warning: return MSG_TYPE_WARNING; + case MsgType::error: return MSG_TYPE_ERROR; + //*INDENT-ON* + } + assert(false); + return MSG_TYPE_ERROR; + }()); requestUiUpdate(false /*force*/); //throw AbortProcess } diff --git a/FreeFileSync/Source/ui/gui_status_handler.h b/FreeFileSync/Source/ui/gui_status_handler.h index 1c75c2b4..beb4d558 100644 --- a/FreeFileSync/Source/ui/gui_status_handler.h +++ b/FreeFileSync/Source/ui/gui_status_handler.h @@ -31,7 +31,7 @@ public: ~StatusHandlerTemporaryPanel(); void initNewPhase (int itemsTotal, int64_t bytesTotal, ProcessPhase phaseID) override; // - void logInfo (const std::wstring& msg) override; // + void logMessage (const std::wstring& msg, MsgType type) override; // void reportWarning (const std::wstring& msg, bool& warningActive) override; //throw AbortProcess Response reportError (const ErrorInfo& errorInfo) override; // void reportFatalError(const std::wstring& msg) override; // @@ -79,7 +79,7 @@ public: ~StatusHandlerFloatingDialog(); void initNewPhase (int itemsTotal, int64_t bytesTotal, ProcessPhase phaseID) override; // - void logInfo (const std::wstring& msg) override; // + void logMessage (const std::wstring& msg, MsgType type) override; // void reportWarning (const std::wstring& msg, bool& warningActive) override; //throw AbortProcess Response reportError (const ErrorInfo& errorInfo) override; // void reportFatalError(const std::wstring& msg) override; // diff --git a/FreeFileSync/Source/ui/main_dlg.cpp b/FreeFileSync/Source/ui/main_dlg.cpp index 82bbc8e5..d3e74bb2 100644 --- a/FreeFileSync/Source/ui/main_dlg.cpp +++ b/FreeFileSync/Source/ui/main_dlg.cpp @@ -14,6 +14,7 @@ #include <zen/shutdown.h> #include <zen/resolve_path.h> #include <wx/clipbrd.h> +#include <wx/colordlg.h> #include <wx/wupdlock.h> #include <wx/sound.h> #include <wx/filedlg.h> @@ -3083,7 +3084,8 @@ void MainDialog::onConfigSave(wxCommandEvent& event) trySaveBatchConfig(&activeCfgFilePath); else showNotificationDialog(this, DialogInfoType::error, - PopupDialogCfg().setDetailInstructions(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(activeCfgFilePath)))); + PopupDialogCfg().setDetailInstructions(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(activeCfgFilePath)) + + L"\n\n" + _("Unexpected file extension:") + L' ' + fmtPath(getFileExtension(activeCfgFilePath)))); } } @@ -3256,7 +3258,8 @@ bool MainDialog::saveOldConfig() //"false": error/cancel else { showNotificationDialog(this, DialogInfoType::error, - PopupDialogCfg().setDetailInstructions(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(activeCfgFilePath)))); + PopupDialogCfg().setDetailInstructions(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(activeCfgFilePath)) + + L"\n\n" + _("Unexpected file extension:") + L' ' + fmtPath(getFileExtension(activeCfgFilePath)) )); return false; } break; @@ -3396,37 +3399,39 @@ void MainDialog::removeSelectedCfgHistoryItems(bool deleteFromDisk) if (deleteFromDisk) { + //=========================================================================== + FocusPreserver fp; + std::wstring fileList; for (const Zstring& filePath : filePaths) fileList += utfTo<std::wstring>(filePath) + L'\n'; - FocusPreserver fp; - bool moveToRecycler = true; if (showDeleteDialog(this, fileList, static_cast<int>(filePaths.size()), moveToRecycler) != ConfirmationButton::accept) return; - std::vector<Zstring> deletedPaths; - std::optional<FileError> firstError; - - for (const Zstring& filePath : filePaths) - try - { - AbstractPath cfgPath = createItemPathNative(filePath); + disableGuiElements(true /*enableAbort*/); //StatusHandlerTemporaryPanel will internally process Window messages, so avoid unexpected callbacks! + auto app = wxTheApp; //fix lambda/wxWigets/VC fuck up + ZEN_ON_SCOPE_EXIT(app->Yield(); enableGuiElements()); //ui update before enabling buttons again: prevent strange behaviour of delayed button clicks - if (moveToRecycler) - AFS::recycleItemIfExists(cfgPath); //throw FileError - else - AFS::removeFileIfExists(cfgPath); //throw FileError + StatusHandlerTemporaryPanel statusHandler(*this, std::chrono::system_clock::now() /*startTime*/, + false /*ignoreErrors*/, + 0 /*autoRetryCount*/, + std::chrono::seconds(0) /*autoRetryDelay*/, + globalCfg_.soundFileAlertPending); + std::vector<Zstring> deletedPaths; + try + { + deleteListOfFiles(filePaths, deletedPaths, moveToRecycler, globalCfg_.warnDlgs.warnRecyclerMissing, statusHandler); //throw AbortProcess + } + catch (AbortProcess&) {} - deletedPaths.push_back(filePath); - } - catch (const FileError& e) { if (!firstError) firstError = e; } + const StatusHandlerTemporaryPanel::Result r = statusHandler.reportResults(); //noexcept + setLastOperationLog(r.summary, r.errorLog.ptr()); - if (firstError) - showNotificationDialog(this, DialogInfoType::error, PopupDialogCfg().setDetailInstructions(firstError->toString())); filePaths = deletedPaths; + //=========================================================================== } cfggrid::getDataView(*m_gridCfgHistory).removeItems(filePaths); @@ -3557,6 +3562,13 @@ void MainDialog::onCfgGridContext(GridContextMenuEvent& event) ContextMenu menu; const std::vector<size_t> selectedRows = m_gridCfgHistory->getSelectedRows(); + std::vector<Zstring> cfgFilePaths; + for (size_t row : selectedRows) + if (const ConfigView::Details* cfg = cfggrid::getDataView(*m_gridCfgHistory).getItem(row)) + cfgFilePaths.push_back(cfg->cfgItem.cfgFilePath); + else + assert(false); + //-------------------------------------------------------------------------------------------------------- const bool renameEnabled = [&] { @@ -3569,24 +3581,17 @@ void MainDialog::onCfgGridContext(GridContextMenuEvent& event) //-------------------------------------------------------------------------------------------------------- ContextMenu submenu; - auto addColorOption = [&](const wxColor& col, const wxString& name) + auto applyBackColor = [this, &cfgFilePaths](const wxColor& col) { - auto applyBackColor = [this, col, &selectedRows] - { - std::vector<Zstring> filePaths; - for (size_t row : selectedRows) - if (const ConfigView::Details* cfg = cfggrid::getDataView(*m_gridCfgHistory).getItem(row)) - filePaths.push_back(cfg->cfgItem.cfgFilePath); - else - assert(false); + cfggrid::getDataView(*m_gridCfgHistory).setBackColor(cfgFilePaths, col); - cfggrid::getDataView(*m_gridCfgHistory).setBackColor(filePaths, col); - - //re-apply selection (after sorting by color tags): - cfggrid::addAndSelect(*m_gridCfgHistory, activeConfigFiles_, false /*scrollToSelection*/); - //m_gridCfgHistory->Refresh(); <- implicit in last call - }; + //re-apply selection (after sorting by color tags): + cfggrid::addAndSelect(*m_gridCfgHistory, activeConfigFiles_, false /*scrollToSelection*/); + //m_gridCfgHistory->Refresh(); <- implicit in last call + }; + auto addColorOption = [&](const wxColor& col, const wxString& name) + { wxBitmap bmpSquare(this->GetCharHeight(), this->GetCharHeight()); //seems we don't need to pass 24-bit depth here even for high-contrast color schemes bmpSquare.SetScaleFactor(getDisplayScaleFactor()); { @@ -3595,16 +3600,88 @@ void MainDialog::onCfgGridContext(GridContextMenuEvent& event) const wxColor fillCol = col.Ok() ? col : wxSystemSettings::GetColour(wxSYS_COLOUR_WINDOW); drawInsetRectangle(dc, wxRect(bmpSquare.GetSize()), fastFromDIP(1), borderCol, fillCol); } - submenu.addItem(name, applyBackColor, bmpSquare.ConvertToImage(), !selectedRows.empty()); + submenu.addItem(name, [&, col] { applyBackColor(col); }, bmpSquare.ConvertToImage(), !selectedRows.empty()); + }; + + const std::vector<std::pair<wxColor, wxString>> defaultColors + { + {wxNullColour /*=> !wxColor::IsOk()*/, L'(' + _("&Default") + L')'}, //meta options should be enclosed in parentheses + {{0xff, 0xd8, 0xcb}, _("Red")}, + {{0xff, 0xf9, 0x99}, _("Yellow")}, + {{0xcc, 0xff, 0x99}, _("Green")}, + {{0xcc, 0xff, 0xff}, _("Cyan")}, + {{0xcc, 0xcc, 0xff}, _("Blue")}, + {{0xf2, 0xcb, 0xff}, _("Purple")}, + {{0xdd, 0xdd, 0xdd}, _("Gray")}, }; - addColorOption(wxNullColour, L'(' + _("&Default") + L')'); //meta options should be enclosed in parentheses - addColorOption({0xff, 0xd8, 0xcb}, _("Red")); - addColorOption({0xff, 0xf9, 0x99}, _("Yellow")); - addColorOption({0xcc, 0xff, 0x99}, _("Green")); - addColorOption({0xcc, 0xff, 0xff}, _("Cyan")); - addColorOption({0xcc, 0xcc, 0xff}, _("Blue")); - addColorOption({0xf2, 0xcb, 0xff}, _("Purple")); - addColorOption({0xdd, 0xdd, 0xdd}, _("Grey")); + std::unordered_set<wxUint32> addedColorCodes; + + //add default colors + for (const auto& [color, name] : defaultColors) + { + addColorOption(color, name); + if (color.IsOk()) + addedColorCodes.insert(color.GetRGBA()); + } + + //show color picker + wxBitmap bmpColorPicker(this->GetCharHeight(), this->GetCharHeight()); //seems we don't need to pass 24-bit depth here even for high-contrast color schemes + bmpColorPicker.SetScaleFactor(getDisplayScaleFactor()); + { + wxMemoryDC dc(bmpColorPicker); + const wxColor borderCol(0xdd, 0xdd, 0xdd); //light grey + drawInsetRectangle(dc, wxRect(bmpColorPicker.GetSize()), fastFromDIP(1), borderCol, wxSystemSettings::GetColour(wxSYS_COLOUR_WINDOW)); + + dc.SetFont(dc.GetFont().Bold()); + dc.DrawText(L"?", wxPoint() + (bmpColorPicker.GetSize() - dc.GetTextExtent(L"?")) / 2); + } + + submenu.addItem(_("Select different color..."), [&] + { + wxColourData colCfg; + colCfg.SetChooseFull(true); + colCfg.SetChooseAlpha(false); + colCfg.SetColour(defaultColors[1].first); //tentative + + if (const ConfigView::Details* cfg = cfggrid::getDataView(*m_gridCfgHistory).getItem(selectedRows[0])) + if (cfg->cfgItem.backColor.IsOk()) + colCfg.SetColour(cfg->cfgItem.backColor); + + int i = 0; + for (const auto& [color, name] : defaultColors) + if (color.IsOk() && i < static_cast<int>(wxColourData::NUM_CUSTOM)) + colCfg.SetCustomColour(i++, color); + + auto fixColorPickerColor = [](const wxColor& col) + { + assert(col.Alpha() == 255); + return col; + }; + wxColourDialog dlg(this, &colCfg); + dlg.Center(); + + dlg.Bind(wxEVT_COLOUR_CHANGED, [&](wxColourDialogEvent& event2) + { + //show preview during color selection (Windows-only atm) + cfggrid::getDataView(*m_gridCfgHistory).setBackColor(cfgFilePaths, fixColorPickerColor(event2.GetColour()), true /*previewOnly*/); + m_gridCfgHistory->Refresh(); + }); + + if (dlg.ShowModal() == wxID_OK) + applyBackColor(fixColorPickerColor(dlg.GetColourData().GetColour())); + else //shut off color preview + { + cfggrid::getDataView(*m_gridCfgHistory).setBackColor(cfgFilePaths, wxNullColour, true /*previewOnly*/); + m_gridCfgHistory->Refresh(); + } + }, bmpColorPicker.ConvertToImage()); + + //add user-defined colors + for (const ConfigFileItem& item : cfggrid::getDataView(*m_gridCfgHistory).get()) + if (item.backColor.IsOk()) + if (const auto [it, inserted] = addedColorCodes.insert(item.backColor.GetRGBA()); + inserted) + addColorOption(item.backColor, item.backColor.GetAsString(wxC2S_HTML_SYNTAX)); //#RRGGBB menu.addSubmenu(_("Background color"), submenu, loadImage("color_sicon"), !selectedRows.empty()); menu.addSeparator(); @@ -4194,7 +4271,8 @@ void MainDialog::onCompare(wxCommandEvent& event) st.updateCount() + st.deleteCount() == 0) { - setStatusInfo(_("No files to synchronize"), true /*highlight*/); //don't flashStatusInfo() + setStatusInfo(_("No files to synchronize"), true /*highlight*/); //user might be AFK: don't flashStatusInfo() + //overwrites status info already set in updateGui() above updateConfigLastRunStats(std::chrono::system_clock::to_time_t(r.summary.startTime), r.summary.syncResult, getNullPath() /*logFilePath*/); } @@ -4300,6 +4378,7 @@ void MainDialog::updateStatistics() setIntValue(*m_staticTextUpdateRight, st.updateCount<SelectSide::right>(), *m_bitmapUpdateRight, "so_update_right_sicon"); setIntValue(*m_staticTextDeleteRight, st.deleteCount<SelectSide::right>(), *m_bitmapDeleteRight, "so_delete_right_sicon"); + m_panelViewFilter->Layout(); //[!] statistics panel size changed, so this is needed m_panelStatistics->Layout(); m_panelStatistics->Refresh(); //fix small mess up on RTL layout } @@ -5038,7 +5117,7 @@ void MainDialog::updateGridViewData() m_bpButtonViewType ->Show(anyViewButtonShown); m_bpButtonViewFilterContext->Show(anyViewButtonShown); - m_panelViewFilter->Layout(); + //m_panelViewFilter->Layout(); -> yes, needed, but will also be called in updateStatistics(); //all three grids retrieve their data directly via gridDataView filegrid::refresh(*m_gridMainL, *m_gridMainC, *m_gridMainR); @@ -5689,41 +5768,38 @@ void MainDialog::onMenuExportFileList(wxCommandEvent& event) const Zstring csvFilePath = appendPath(tempFileBuf_.getAndCreateFolderPath(), //throw FileError title + Zstr("~") + shortGuid + Zstr(".csv")); + const Zstring tmpFilePath = getPathWithTempName(csvFilePath); + + FileOutputBuffered tmpFile(tmpFilePath, nullptr /*notifyUnbufferedIO*/); //throw FileError, (ErrorTargetExisting) + + auto writeString = [&](const std::string& str) { tmpFile.write(str.data(), str.size()); }; //throw FileError - TempFileOutput fileOut(csvFilePath, nullptr /*notifyUnbufferedIO*/); //throw FileError + //main grid: write rows one after the other instead of creating one big string: memory allocation might fail; think 1 million rows! + writeString(header); //throw FileError - fileOut.write(&header[0], header.size()); //throw FileError, (X) - /* main grid: write rows one after the other instead of creating one big string: memory allocation might fail; think 1 million rows! - performance test case "export 600.000 rows" to CSV: - aproach 1. assemble single temporary string, then write file: 4.6s - aproach 2. write to buffered file output directly for each row: 6.4s */ - std::string buffer; const size_t rowCount = m_gridMainL->getRowCount(); for (size_t row = 0; row < rowCount; ++row) { for (const Grid::ColAttributes& ca : colAttrLeft) - { - buffer += fmtValue(provLeft->getValue(row, ca.type)); - buffer += CSV_SEP; - } + writeString(fmtValue(provLeft->getValue(row, ca.type)) += CSV_SEP); //throw FileError for (const Grid::ColAttributes& ca : colAttrCenter) - { - buffer += fmtValue(provCenter->getValue(row, ca.type)); - buffer += CSV_SEP; - } + writeString(fmtValue(provCenter->getValue(row, ca.type)) += CSV_SEP); //throw FileError for (const Grid::ColAttributes& ca : colAttrRight) - { - buffer += fmtValue(provRight->getValue(row, ca.type)); - buffer += CSV_SEP; - } - buffer += LINE_BREAK; + writeString(fmtValue(provRight->getValue(row, ca.type)) += CSV_SEP); //throw FileError - fileOut.write(&buffer[0], buffer.size()); //throw FileError, (X) - buffer.clear(); + writeString(LINE_BREAK); //throw FileError } - fileOut.commit(); //throw FileError, (X) + + tmpFile.finalize(); //throw FileError + //take over ownership: + ZEN_ON_SCOPE_FAIL( try { removeFilePlain(tmpFilePath); /*throw FileError*/ } + catch (FileError&) {}); + warn_static("log it!") + + //operation finished: move temp file transactionally + moveAndRenameItem(tmpFilePath, csvFilePath, true /*replaceExisting*/); //throw FileError, (ErrorMoveUnsupported), (ErrorTargetExisting) openWithDefaultApp(csvFilePath); //throw FileError diff --git a/FreeFileSync/Source/ui/small_dlgs.cpp b/FreeFileSync/Source/ui/small_dlgs.cpp index 34bf261d..76461a90 100644 --- a/FreeFileSync/Source/ui/small_dlgs.cpp +++ b/FreeFileSync/Source/ui/small_dlgs.cpp @@ -21,7 +21,6 @@ #include <wx+/no_flicker.h> #include <wx+/image_tools.h> #include <wx+/font_size.h> -//#include <wx+/std_button_layout.h> #include <wx+/popup_dlg.h> #include <wx+/async_task.h> #include <wx+/image_resources.h> @@ -39,7 +38,6 @@ #include "../base/icon_loader.h" #include "../status_handler.h" //uiUpdateDue() #include "../version/version.h" -//#include "../log_file.h" #include "../ffs_paths.h" #include "../icon_buffer.h" @@ -148,8 +146,8 @@ AboutDlg::AboutDlg(wxWindow* parent) : AboutDlgGenerated(parent) GetSizer()->SetSizeHints(this); //~=Fit() + SetMinSize() { - const int imageWidth = (m_panelDonate->GetSize().GetWidth() - 5 - 5 /* grey border*/) / 2; - const int textWidth = m_panelDonate->GetSize().GetWidth() - 5 - 5 - imageWidth; + const int imageWidth = (m_panelDonate->GetSize().GetWidth() - 5 - 5 - 5 /* grey border*/) / 2; + const int textWidth = m_panelDonate->GetSize().GetWidth() - 5 - 5 - 5 - imageWidth; setImage(*m_bitmapAnimalSmall, shrinkImage(animalImg, imageWidth, -1 /*maxHeight*/)); @@ -1223,10 +1221,8 @@ private: []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnNotEnoughDiskSpace = show; }, _("Not enough free disk space available in:")}, {[](const XmlGlobalSettings& gs){ return gs.warnDlgs.warnUnresolvedConflicts; }, []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnUnresolvedConflicts = show; }, _("The following items have unresolved conflicts and will not be synchronized:")}, - {[](const XmlGlobalSettings& gs){ return gs.warnDlgs.warnModificationTimeError; }, - []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnModificationTimeError = show; }, _("Cannot write modification time of %x.")}, {[](const XmlGlobalSettings& gs){ return gs.warnDlgs.warnRecyclerMissing; }, - []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnRecyclerMissing = show; }, _("The recycle bin is not supported by the following folders. Deleted or overwritten files will not be able to be restored:")}, + []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnRecyclerMissing = show; }, _("The recycle bin is not available for %x.") + L' ' + _("Ignore and delete permanently each time recycle bin is unavailable?")}, {[](const XmlGlobalSettings& gs){ return gs.warnDlgs.warnInputFieldEmpty; }, []( XmlGlobalSettings& gs, bool show){ gs.warnDlgs.warnInputFieldEmpty = show; }, _("A folder input field is empty.") + L' ' + _("The corresponding folder will be considered as empty.")}, {[](const XmlGlobalSettings& gs){ return gs.warnDlgs.warnDirectoryLockFailed; }, @@ -1453,12 +1449,13 @@ void OptionsDlg::playSoundWithDiagnostics(const wxString& filePath) { try { - //::PlaySound() => NO failure indication on Windows! does not set last error! - //wxSound::Play(..., wxSOUND_SYNC) can return false, but does not provide details! + //::PlaySound() on Windows does not set last error! + //wxSound::Play(..., wxSOUND_SYNC) can return false, but also without details! //=> check file access manually: [[maybe_unused]] const std::string& stream = getFileContent(utfTo<Zstring>(filePath), nullptr /*notifyUnbufferedIO*/); //throw FileError - [[maybe_unused]] const bool success = wxSound::Play(filePath, wxSOUND_ASYNC); + if (!wxSound::Play(filePath, wxSOUND_ASYNC)) + throw FileError(L"Sound playback failed. No further diagnostics available."); } catch (const FileError& e) { showNotificationDialog(this, DialogInfoType::error, PopupDialogCfg().setDetailInstructions(e.toString())); } } @@ -1959,7 +1956,7 @@ DownloadProgressWindow::~DownloadProgressWindow() { pimpl_->Destroy(); } void DownloadProgressWindow::notifyNewFile(const Zstring& filePath) { pimpl_->notifyNewFile(filePath); } void DownloadProgressWindow::notifyProgress(int64_t delta) { pimpl_->notifyProgress(delta); } -void DownloadProgressWindow::requestUiUpdate() { pimpl_->requestUiUpdate(); } //throw CancelPressed +void DownloadProgressWindow::requestUiUpdate() { pimpl_->requestUiUpdate(); } //throw CancelPressed //######################################################################################## diff --git a/FreeFileSync/Source/ui/sync_cfg.cpp b/FreeFileSync/Source/ui/sync_cfg.cpp index 14f78eb8..dfdb4bc0 100644 --- a/FreeFileSync/Source/ui/sync_cfg.cpp +++ b/FreeFileSync/Source/ui/sync_cfg.cpp @@ -299,9 +299,9 @@ private: void onDifferent (wxCommandEvent& event) override; void onConflict (wxCommandEvent& event) override; - void onDeletionPermanent (wxCommandEvent& event) override { handleDeletion_ = DeletionPolicy::permanent; updateSyncGui(); } - void onDeletionRecycler (wxCommandEvent& event) override { handleDeletion_ = DeletionPolicy::recycler; updateSyncGui(); } - void onDeletionVersioning (wxCommandEvent& event) override { handleDeletion_ = DeletionPolicy::versioning; updateSyncGui(); } + void onDeletionPermanent (wxCommandEvent& event) override { deletionVariant_ = DeletionVariant::permanent; updateSyncGui(); } + void onDeletionRecycler (wxCommandEvent& event) override { deletionVariant_ = DeletionVariant::recycler; updateSyncGui(); } + void onDeletionVersioning (wxCommandEvent& event) override { deletionVariant_ = DeletionVariant::versioning; updateSyncGui(); } void onToggleMiscOption(wxCommandEvent& event) override { updateMiscGui(); } void onToggleMiscEmail (wxCommandEvent& event) override @@ -323,7 +323,7 @@ private: //parameters with ownership NOT within GUI controls! SyncDirectionConfig directionCfg_; - DeletionPolicy handleDeletion_ = DeletionPolicy::recycler; //use Recycler, delete permanently or move to user-defined location + DeletionVariant deletionVariant_ = DeletionVariant::recycler; //use Recycler, delete permanently or move to user-defined location const std::function<size_t(const Zstring& folderPathPhrase)> getDeviceParallelOps_; const std::function<void (const Zstring& folderPathPhrase, size_t parallelOps)> setDeviceParallelOps_; @@ -666,15 +666,16 @@ globalLogFolderPhrase_(globalLogFolderPhrase) m_staticTextFolderPairLabel->Hide(); } - //temporarily set main config as reference for window height calculations: + //temporarily set main config as reference for window min size calculations: globalPairCfg_ = GlobalPairConfig(); - globalPairCfg_.syncCfg.directionCfg.var = SyncVariant::mirror; // - globalPairCfg_.syncCfg.handleDeletion = DeletionPolicy::versioning; // - globalPairCfg_.syncCfg.versioningFolderPhrase = Zstr("dummy"); //set tentatively for sync dir height calculation below - globalPairCfg_.syncCfg.versioningStyle = VersioningStyle::timestampFile; // - globalPairCfg_.syncCfg.versionMaxAgeDays = 30; // - globalPairCfg_.miscCfg.altLogFolderPathPhrase = Zstr("dummy"); // - globalPairCfg_.miscCfg.emailNotifyAddress = "dummy"; // + globalPairCfg_.syncCfg.directionCfg.var = SyncVariant::mirror; + globalPairCfg_.syncCfg.deletionVariant = DeletionVariant::versioning; + globalPairCfg_.syncCfg.versioningFolderPhrase = Zstr("dummy"); + globalPairCfg_.syncCfg.versioningStyle = VersioningStyle::timestampFile; + globalPairCfg_.syncCfg.versionMaxAgeDays = 30; + globalPairCfg_.miscCfg.autoRetryCount = 1; + globalPairCfg_.miscCfg.altLogFolderPathPhrase = Zstr("dummy"); + globalPairCfg_.miscCfg.emailNotifyAddress = "dummy"; selectFolderPairConfig(-1); @@ -1191,7 +1192,7 @@ std::optional<SyncConfig> ConfigDialog::getSyncConfig() const SyncConfig syncCfg; syncCfg.directionCfg = directionCfg_; - syncCfg.handleDeletion = handleDeletion_; + syncCfg.deletionVariant = deletionVariant_; syncCfg.versioningFolderPhrase = versioningFolder_.getPath(); syncCfg.versioningStyle = getEnumVal(enumVersioningStyle_, *m_choiceVersioningStyle); if (syncCfg.versioningStyle != VersioningStyle::replace) @@ -1212,7 +1213,7 @@ void ConfigDialog::setSyncConfig(const SyncConfig* syncCfg) const SyncConfig tmpCfg = syncCfg ? *syncCfg : globalPairCfg_.syncCfg; directionCfg_ = tmpCfg.directionCfg; //make working copy; ownership *not* on GUI - handleDeletion_ = tmpCfg.handleDeletion; + deletionVariant_ = tmpCfg.deletionVariant; versioningFolder_.setPath(tmpCfg.versioningFolderPhrase); setEnumVal(enumVersioningStyle_, *m_choiceVersioningStyle, tmpCfg.versioningStyle); @@ -1281,13 +1282,13 @@ void ConfigDialog::updateSyncGui() m_buttonCustom->setActive(SyncVariant::custom == directionCfg_.var && syncOptionsEnabled); //syncOptionsEnabled: nudge wxWidgets to render inactive config state (needed on Windows, NOT on Linux!) - m_buttonRecycler ->setActive(DeletionPolicy::recycler == handleDeletion_ && syncOptionsEnabled); - m_buttonPermanent ->setActive(DeletionPolicy::permanent == handleDeletion_ && syncOptionsEnabled); - m_buttonVersioning->setActive(DeletionPolicy::versioning == handleDeletion_ && syncOptionsEnabled); + m_buttonRecycler ->setActive(DeletionVariant::recycler == deletionVariant_ && syncOptionsEnabled); + m_buttonPermanent ->setActive(DeletionVariant::permanent == deletionVariant_ && syncOptionsEnabled); + m_buttonVersioning->setActive(DeletionVariant::versioning == deletionVariant_ && syncOptionsEnabled); - switch (handleDeletion_) //unconditionally update image, including "local options off" + switch (deletionVariant_) //unconditionally update image, including "local options off" { - case DeletionPolicy::recycler: + case DeletionVariant::recycler: { wxImage imgTrash = loadImage("delete_recycler"); //use system icon if available (can fail on Linux??) @@ -1298,17 +1299,17 @@ void ConfigDialog::updateSyncGui() setText(*m_staticTextDeletionTypeDescription, _("Retain deleted and overwritten files in the recycle bin")); } break; - case DeletionPolicy::permanent: + case DeletionVariant::permanent: setImage(*m_bitmapDeletionType, greyScaleIfDisabled(loadImage("delete_permanently"), syncOptionsEnabled)); setText(*m_staticTextDeletionTypeDescription, _("Delete and overwrite files permanently")); break; - case DeletionPolicy::versioning: + case DeletionVariant::versioning: setImage(*m_bitmapVersioning, greyScaleIfDisabled(loadImage("delete_versioning"), syncOptionsEnabled)); break; } //m_staticTextDeletionTypeDescription->Wrap(fastFromDIP(200)); //needs to be reapplied after SetLabel() - const bool versioningSelected = handleDeletion_ == DeletionPolicy::versioning; + const bool versioningSelected = deletionVariant_ == DeletionVariant::versioning; m_bitmapDeletionType ->Show(!versioningSelected); m_staticTextDeletionTypeDescription->Show(!versioningSelected); @@ -1575,14 +1576,9 @@ void ConfigDialog::selectFolderPairConfig(int newPairIndexToShow) bSizerSyncMisc ->Show(mainConfigSelected); if (mainConfigSelected) + { m_hyperlinkPerfDeRequired->Show(!enableExtraFeatures_); //keep after bSizerPerformance->Show() - m_panelCompSettingsTab ->Layout(); //fix comp panel glitch on Win 7 125% font size + perf panel - m_panelFilterSettingsTab->Layout(); - m_panelSyncSettingsTab ->Layout(); - - if (mainConfigSelected) - { //update the devices list for "parallel file operations" before calling setMiscSyncOptions(): // => should be enough to do this when selecting the main config // => to be "perfect" we'd have to update already when the user drags & drops a different versioning folder @@ -1598,10 +1594,10 @@ void ConfigDialog::selectFolderPairConfig(int newPairIndexToShow) addDevicePath(fpCfg.folderPathPhraseLeft); addDevicePath(fpCfg.folderPathPhraseRight); - if (fpCfg.localSyncCfg && fpCfg.localSyncCfg->handleDeletion == DeletionPolicy::versioning) + if (fpCfg.localSyncCfg && fpCfg.localSyncCfg->deletionVariant == DeletionVariant::versioning) addDevicePath(fpCfg.localSyncCfg->versioningFolderPhrase); } - if (globalPairCfg_.syncCfg.handleDeletion == DeletionPolicy::versioning) //let's always add, even if *all* folder pairs use a local sync config (=> strange!) + if (globalPairCfg_.syncCfg.deletionVariant == DeletionVariant::versioning) //let's always add, even if *all* folder pairs use a local sync config (=> strange!) addDevicePath(globalPairCfg_.syncCfg.versioningFolderPhrase); //--------------------------------------------------------------------------------------------------------------- @@ -1616,6 +1612,10 @@ void ConfigDialog::selectFolderPairConfig(int newPairIndexToShow) setSyncConfig(get(localPairCfg_[selectedPairIndexToShow_].localSyncCfg)); setFilterConfig (localPairCfg_[selectedPairIndexToShow_].localFilter); } + + m_panelCompSettingsTab ->Layout(); //fix comp panel glitch on Win 7 125% font size + perf panel + m_panelFilterSettingsTab->Layout(); + m_panelSyncSettingsTab ->Layout(); } @@ -1642,7 +1642,7 @@ bool ConfigDialog::unselectFolderPairConfig(bool validateParams) return false; } - if (syncCfg && syncCfg->handleDeletion == DeletionPolicy::versioning) + if (syncCfg && syncCfg->deletionVariant == DeletionVariant::versioning) { if (AFS::isNullPath(createAbstractPath(syncCfg->versioningFolderPhrase))) { diff --git a/FreeFileSync/Source/ui/tray_icon.cpp b/FreeFileSync/Source/ui/tray_icon.cpp index 6b899a82..8473079b 100644 --- a/FreeFileSync/Source/ui/tray_icon.cpp +++ b/FreeFileSync/Source/ui/tray_icon.cpp @@ -180,7 +180,7 @@ FfsTrayIcon::FfsTrayIcon(const std::function<void()>& requestResume) : iconGenerator_(std::make_unique<ProgressIconGenerator>(loadImage("FFS_tray_24"))) { [[maybe_unused]] const bool rv = trayIcon_->SetIcon(iconGenerator_->get(activeFraction_), activeToolTip_); - //caveat wxTaskBarIcon::SetIcon() can return true, even if not wxTaskBarIcon::IsAvailable()!!! + assert(rv); //caveat wxTaskBarIcon::SetIcon() can return true, even if not wxTaskBarIcon::IsAvailable()!!! } diff --git a/FreeFileSync/Source/ui/version_check.cpp b/FreeFileSync/Source/ui/version_check.cpp index 3e4f80cb..a3656012 100644 --- a/FreeFileSync/Source/ui/version_check.cpp +++ b/FreeFileSync/Source/ui/version_check.cpp @@ -93,10 +93,13 @@ std::wstring getIso639Language() std::wstring localeName(wxLocale::GetLanguageCanonicalName(wxLocale::GetSystemLanguage())); localeName = beforeFirst(localeName, L'@', IfNotFoundReturn::all); //the locale may contain an @, e.g. "sr_RS@latin"; see wxLocale::InitLanguagesDB() - assert(beforeFirst(localeName, L'_', IfNotFoundReturn::all).size() == 2); if (!localeName.empty()) - return beforeFirst(localeName, L'_', IfNotFoundReturn::all); + { + const std::wstring langCode = beforeFirst(localeName, L'_', IfNotFoundReturn::all); + assert(langCode.size() == 2 || langCode.size() == 3); //ISO 639: 3-letter possible! + return langCode; + } assert(false); return L"zz"; } @@ -112,7 +115,11 @@ std::wstring getIso3166Country() localeName = beforeFirst(localeName, L'@', IfNotFoundReturn::all); //the locale may contain an @, e.g. "sr_RS@latin"; see wxLocale::InitLanguagesDB() if (contains(localeName, L'_')) - return afterFirst(localeName, L'_', IfNotFoundReturn::none); + { + const std::wstring cc = afterFirst(localeName, L'_', IfNotFoundReturn::none); + assert(cc.size() == 2 || cc.size() == 3); //ISO 3166: 3-letter possible! + return cc; + } assert(false); return L"ZZ"; } @@ -169,7 +176,7 @@ void showUpdateAvailableDialog(wxWindow* parent, const std::string& onlineVersio try { updateDetailsMsg = utfTo<std::wstring>(sendHttpGet(utfTo<Zstring>("https://api.freefilesync.org/latest_changes?" + xWwwFormUrlEncode({{"since", ffsVersion}})), - ffsUpdateCheckUserAgent, Zstring() /*caCertFilePath*/, nullptr /*notifyUnbufferedIO*/).readAll()); //throw SysError + ffsUpdateCheckUserAgent, Zstring() /*caCertFilePath*/).readAll(nullptr /*notifyUnbufferedIO*/)); //throw SysError } catch (const SysError& e) { updateDetailsMsg = _("Failed to retrieve update information.") + + L"\n\n" + e.toString(); } @@ -181,7 +188,7 @@ void showUpdateAvailableDialog(wxWindow* parent, const std::string& onlineVersio setDetailInstructions(updateDetailsMsg), _("&Download"))) { case ConfirmationButton::accept: //download - openBrowserForDownload(); + openBrowserForDownload(parent); break; case ConfirmationButton::cancel: break; @@ -191,8 +198,8 @@ void showUpdateAvailableDialog(wxWindow* parent, const std::string& onlineVersio std::string getOnlineVersion(const std::vector<std::pair<std::string, std::string>>& postParams) //throw SysError { - const std::string response = sendHttpPost(Zstr("https://api.freefilesync.org/latest_version"), postParams, - ffsUpdateCheckUserAgent, Zstring() /*caCertFilePath*/, nullptr /*notifyUnbufferedIO*/).readAll(); //throw SysError + const std::string response = sendHttpPost(Zstr("https://api.freefilesync.org/latest_version"), postParams, nullptr /*notifyUnbufferedIO*/, + ffsUpdateCheckUserAgent, Zstring() /*caCertFilePath*/).readAll(nullptr /*notifyUnbufferedIO*/); //throw SysError if (response.empty() || !std::all_of(response.begin(), response.end(), [](char c) { return isDigit(c) || c == FFS_VERSION_SEPARATOR; }) || diff --git a/FreeFileSync/Source/version/version.h b/FreeFileSync/Source/version/version.h index 961e8c9a..1ac5f576 100644 --- a/FreeFileSync/Source/version/version.h +++ b/FreeFileSync/Source/version/version.h @@ -3,7 +3,7 @@ namespace fff { -const char ffsVersion[] = "11.25"; //internal linkage! +const char ffsVersion[] = "11.26"; //internal linkage! const char FFS_VERSION_SEPARATOR = '.'; } diff --git a/libcurl/curl_wrap.cpp b/libcurl/curl_wrap.cpp index 61c36404..05fedb57 100644 --- a/libcurl/curl_wrap.cpp +++ b/libcurl/curl_wrap.cpp @@ -33,6 +33,8 @@ void zen::libcurlInit() [[maybe_unused]] const CURLcode rc2 = ::curl_global_init(CURL_GLOBAL_NOTHING /*CURL_GLOBAL_DEFAULT = CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32*/); assert(rc2 == CURLE_OK); + + warn_static("log on error") } @@ -62,8 +64,8 @@ HttpSession::~HttpSession() HttpSession::Result HttpSession::perform(const std::string& serverRelPath, const std::vector<std::string>& extraHeaders, const std::vector<CurlOption>& extraOptions, - const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, // - const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional + const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, //optional + const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; return "bytesToRead" bytes unless end of stream! const std::function<void (const std::string_view& header)>& receiveHeader /*throw X*/, int timeoutSec) //throw SysError, X { @@ -137,11 +139,11 @@ HttpSession::Result HttpSession::perform(const std::string& serverRelPath, //CURLOPT_SSL_VERIFYHOST => //--------------------------------------------------- - auto onHeaderReceived = [&](const void* buffer, size_t len) + auto onHeaderReceived = [&](const char* buffer, size_t len) { try { - receiveHeader({static_cast<const char*>(buffer), len}); //throw X + receiveHeader({buffer, len}); //throw X return len; } catch (...) @@ -150,40 +152,42 @@ HttpSession::Result HttpSession::perform(const std::string& serverRelPath, return len + 1; //signal error condition => CURLE_WRITE_ERROR } }; - using HeaderCbType = decltype(onHeaderReceived); - using HeaderCbWrapperType = size_t (*)(const void* buffer, size_t size, size_t nitems, HeaderCbType* callbackData); //needed for cdecl function pointer cast - HeaderCbWrapperType onHeaderReceivedWrapper = [](const void* buffer, size_t size, size_t nitems, HeaderCbType* callbackData) + curl_write_callback onHeaderReceivedWrapper = [](/*const*/ char* buffer, size_t size, size_t nitems, void* callbackData) { - return (*callbackData)(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda + return (*static_cast<decltype(onHeaderReceived)*>(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; //--------------------------------------------------- - auto onBytesReceived = [&](const void* buffer, size_t len) + auto onBytesReceived = [&](const char* buffer, size_t bytesToWrite) { try { - writeResponse({static_cast<const char*>(buffer), len}); //throw X - return len; + writeResponse({buffer, bytesToWrite}); //throw X + //[!] let's NOT use "incomplete write Posix semantics" for libcurl! + //who knows if libcurl buffers properly, or if it sends incomplete packages!? + return bytesToWrite; } catch (...) { userCallbackException = std::current_exception(); - return len + 1; //signal error condition => CURLE_WRITE_ERROR + return bytesToWrite + 1; //signal error condition => CURLE_WRITE_ERROR } }; - using ReadCbType = decltype(onBytesReceived); - using ReadCbWrapperType = size_t (*)(const void* buffer, size_t size, size_t nitems, ReadCbType* callbackData); //needed for cdecl function pointer cast - ReadCbWrapperType onBytesReceivedWrapper = [](const void* buffer, size_t size, size_t nitems, ReadCbType* callbackData) + curl_write_callback onBytesReceivedWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { - return (*callbackData)(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda + return (*static_cast<decltype(onBytesReceived)*>(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; //--------------------------------------------------- - auto getBytesToSend = [&](void* buffer, size_t len) -> size_t + auto getBytesToSend = [&](char* buffer, size_t bytesToRead) -> size_t { try { - //libcurl calls back until 0 bytes are returned (Posix read() semantics), or, - //if CURLOPT_INFILESIZE_LARGE was set, after exactly this amount of bytes - const size_t bytesRead = readRequest({static_cast<char*>(buffer), len});//throw X; return "bytesToRead" bytes unless end of stream! + /* libcurl calls back until 0 bytes are returned (Posix read() semantics), or, + if CURLOPT_INFILESIZE_LARGE was set, after exactly this amount of bytes + + [!] let's NOT use "incomplete read Posix semantics" for libcurl! + who knows if libcurl buffers properly, or if it requests incomplete packages!? */ + const size_t bytesRead = readRequest({buffer, bytesToRead}); //throw X; return "bytesToRead" bytes unless end of stream + assert(bytesRead == bytesToRead || bytesRead == 0 || readRequest({buffer, bytesToRead}) == 0); return bytesRead; } catch (...) @@ -192,11 +196,9 @@ HttpSession::Result HttpSession::perform(const std::string& serverRelPath, return CURL_READFUNC_ABORT; //signal error condition => CURLE_ABORTED_BY_CALLBACK } }; - using WriteCbType = decltype(getBytesToSend); - using WriteCbWrapperType = size_t (*)(void* buffer, size_t size, size_t nitems, WriteCbType* callbackData); - WriteCbWrapperType getBytesToSendWrapper = [](void* buffer, size_t size, size_t nitems, WriteCbType* callbackData) + curl_read_callback getBytesToSendWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { - return (*callbackData)(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda + return (*static_cast<decltype(getBytesToSend)*>(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; //--------------------------------------------------- if (receiveHeader) @@ -208,6 +210,8 @@ HttpSession::Result HttpSession::perform(const std::string& serverRelPath, { options.emplace_back(CURLOPT_WRITEDATA, &onBytesReceived); options.emplace_back(CURLOPT_WRITEFUNCTION, onBytesReceivedWrapper); + //{CURLOPT_BUFFERSIZE, 256 * 1024} -> defaults is 16 kB which seems to correspond to SSL packet size + //=> setting larget buffers size does nothing (recv still returns only 16 kB) } if (readRequest) { @@ -215,6 +219,7 @@ HttpSession::Result HttpSession::perform(const std::string& serverRelPath, /**/options.emplace_back(CURLOPT_UPLOAD, 1); //issues HTTP PUT options.emplace_back(CURLOPT_READDATA, &getBytesToSend); options.emplace_back(CURLOPT_READFUNCTION, getBytesToSendWrapper); + //{CURLOPT_UPLOAD_BUFFERSIZE, 256 * 1024} -> defaults is 64 kB. apparently no performance improvement for larger buffers like 256 kB } if (std::any_of(extraOptions.begin(), extraOptions.end(), [](const CurlOption& o) { return o.option == CURLOPT_WRITEFUNCTION || o.option == CURLOPT_READFUNCTION; })) @@ -357,7 +362,7 @@ std::wstring zen::formatCurlStatusCode(CURLcode sc) ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_TFTP_UNKNOWNID); ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_REMOTE_FILE_EXISTS); ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_TFTP_NOSUCHUSER); - ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_CONV_FAILED); + ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_OBSOLETE75); ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_OBSOLETE76); ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_SSL_CACERT_BADFILE); ZEN_CHECK_CASE_FOR_CONSTANT(CURLE_REMOTE_FILE_NOT_FOUND); diff --git a/libcurl/curl_wrap.h b/libcurl/curl_wrap.h index 35ee54ba..ea91072c 100644 --- a/libcurl/curl_wrap.h +++ b/libcurl/curl_wrap.h @@ -53,8 +53,8 @@ public: }; Result perform(const std::string& serverRelPath, const std::vector<std::string>& extraHeaders, const std::vector<CurlOption>& extraOptions, - const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, // - const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional + const std::function<void (std::span<const char> buf)>& writeResponse /*throw X*/, //optional + const std::function<size_t(std::span< char> buf)>& readRequest /*throw X*/, //optional; return "bytesToRead" bytes unless end of stream! const std::function<void (const std::string_view& header)>& receiveHeader /*throw X*/, int timeoutSec); //throw SysError, X diff --git a/wx+/graph.cpp b/wx+/graph.cpp index f9094386..bee38e36 100644 --- a/wx+/graph.cpp +++ b/wx+/graph.cpp @@ -730,7 +730,7 @@ void Graph2D::render(wxDC& dc) const //unlike wxDC::DrawRectangle() which just widens inner area! wxDCPenChanger dummy (dc, wxPen(it->second.fillColor, 1 /*[!] width*/)); wxDCBrushChanger dummy2(dc, it->second.fillColor); - dc.DrawPolygon(static_cast<int>(points.size()), &points[0]); + dc.DrawPolygon(static_cast<int>(points.size()), points.data()); } //2. draw all currently set mouse selections (including active selection) diff --git a/wx+/image_resources.cpp b/wx+/image_resources.cpp index 58ae4d25..667b5912 100644 --- a/wx+/image_resources.cpp +++ b/wx+/image_resources.cpp @@ -35,8 +35,8 @@ ImageHolder xbrzScale(int width, int height, const unsigned char* imageRgb, cons //get rid of allocation and buffer std::vector<> at thread-level? => no discernable perf improvement std::vector<uint32_t> buf(hqWidth * hqHeight + width * height); - uint32_t* const argbSrc = &buf[0] + hqWidth * hqHeight; - uint32_t* const xbrTrg = &buf[0]; + uint32_t* const argbSrc = buf.data() + hqWidth * hqHeight; + uint32_t* const xbrTrg = buf.data(); //convert RGB (RGB byte order) to ARGB (BGRA byte order) { diff --git a/zen/base64.h b/zen/base64.h index 48cf2230..cbef7c33 100644 --- a/zen/base64.h +++ b/zen/base64.h @@ -28,8 +28,8 @@ OutputIterator encodeBase64(InputIterator first, InputIterator last, OutputItera template <class InputIterator, class OutputIterator> OutputIterator decodeBase64(InputIterator first, InputIterator last, OutputIterator result); //nothrow! -std::string stringEncodeBase64(const std::string& str); -std::string stringDecodeBase64(const std::string& str); +std::string stringEncodeBase64(const std::string_view& str); +std::string stringDecodeBase64(const std::string_view& str); @@ -156,7 +156,7 @@ OutputIterator decodeBase64(InputIterator first, InputIterator last, OutputItera inline -std::string stringEncodeBase64(const std::string& str) +std::string stringEncodeBase64(const std::string_view& str) { std::string out; encodeBase64(str.begin(), str.end(), std::back_inserter(out)); @@ -165,7 +165,7 @@ std::string stringEncodeBase64(const std::string& str) inline -std::string stringDecodeBase64(const std::string& str) +std::string stringDecodeBase64(const std::string_view& str) { std::string out; decodeBase64(str.begin(), str.end(), std::back_inserter(out)); diff --git a/zen/dir_watcher.cpp b/zen/dir_watcher.cpp index c48928a3..87fa3596 100644 --- a/zen/dir_watcher.cpp +++ b/zen/dir_watcher.cpp @@ -101,13 +101,13 @@ DirWatcher::~DirWatcher() std::vector<DirWatcher::Change> DirWatcher::fetchChanges(const std::function<void()>& requestUiUpdate, std::chrono::milliseconds cbInterval) //throw FileError { - std::vector<std::byte> buffer(512 * (sizeof(inotify_event) + NAME_MAX + 1)); + std::vector<std::byte> buf(512 * (sizeof(inotify_event) + NAME_MAX + 1)); ssize_t bytesRead = 0; do { //non-blocking call, see O_NONBLOCK - bytesRead = ::read(pimpl_->notifDescr, &buffer[0], buffer.size()); + bytesRead = ::read(pimpl_->notifDescr, buf.data(), buf.size()); } while (bytesRead < 0 && errno == EINTR); //"Interrupted function call; When this happens, you should try the call again." @@ -124,7 +124,7 @@ std::vector<DirWatcher::Change> DirWatcher::fetchChanges(const std::function<voi ssize_t bytePos = 0; while (bytePos < bytesRead) { - inotify_event& evt = reinterpret_cast<inotify_event&>(buffer[bytePos]); + inotify_event& evt = reinterpret_cast<inotify_event&>(buf[bytePos]); if (evt.len != 0) //exclude case: deletion of "self", already reported by parent directory watch { diff --git a/zen/file_access.cpp b/zen/file_access.cpp index 2e119e87..1da8bd1b 100644 --- a/zen/file_access.cpp +++ b/zen/file_access.cpp @@ -461,48 +461,47 @@ void zen::copyItemPermissions(const Zstring& sourcePath, const Zstring& targetPa void zen::createDirectory(const Zstring& dirPath) //throw FileError, ErrorTargetExisting { - auto getErrorMsg = [&] { return replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(dirPath)); }; - - //don't allow creating irregular folders! - const Zstring dirName = afterLast(dirPath, FILE_NAME_SEPARATOR, IfNotFoundReturn::all); + try + { + //don't allow creating irregular folders! + const Zstring dirName = afterLast(dirPath, FILE_NAME_SEPARATOR, IfNotFoundReturn::all); - //e.g. "...." https://social.technet.microsoft.com/Forums/windows/en-US/ffee2322-bb6b-4fdf-86f9-8f93cf1fa6cb/ - if (std::all_of(dirName.begin(), dirName.end(), [](Zchar c) { return c == Zstr('.'); })) - /**/throw FileError(getErrorMsg(), replaceCpy<std::wstring>(L"Invalid folder name %x.", L"%x", fmtPath(dirName))); + //e.g. "...." https://social.technet.microsoft.com/Forums/windows/en-US/ffee2322-bb6b-4fdf-86f9-8f93cf1fa6cb/ + if (std::all_of(dirName.begin(), dirName.end(), [](Zchar c) { return c == Zstr('.'); })) + /**/throw SysError(replaceCpy<std::wstring>(L"Invalid folder name %x.", L"%x", fmtPath(dirName))); #if 0 //not appreciated: https://freefilesync.org/forum/viewtopic.php?t=7509 - if (startsWith(dirName, Zstr(' ')) || //Windows can access these just fine once created! - endsWith (dirName, Zstr(' '))) // - throw FileError(getErrorMsg(), replaceCpy<std::wstring>(L"Invalid folder name. %x starts/ends with space character.", L"%x", fmtPath(dirName))); + if (startsWith(dirName, Zstr(' ')) || //Windows can access these just fine once created! + endsWith (dirName, Zstr(' '))) // + throw SysError(replaceCpy<std::wstring>(L"Invalid folder name %x starts/ends with space character.", L"%x", fmtPath(dirName))); #endif + const mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO; //0777 => consider umask! - const mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO; //0777 => consider umask! - - if (::mkdir(dirPath.c_str(), mode) != 0) - { - const int lastError = errno; //copy before directly or indirectly making other system calls! - const std::wstring errorDescr = formatSystemError("mkdir", lastError); - - if (lastError == EEXIST) - throw ErrorTargetExisting(getErrorMsg(), errorDescr); - //else if (lastError == ENOENT) - // throw ErrorTargetPathMissing(errorMsg, errorDescr); - throw FileError(getErrorMsg(), errorDescr); + if (::mkdir(dirPath.c_str(), mode) != 0) + { + const int ec = errno; //copy before directly or indirectly making other system calls! + if (ec == EEXIST) + throw ErrorTargetExisting(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(dirPath)), formatSystemError("mkdir", ec)); + //else if (ec == ENOENT) + // throw ErrorTargetPathMissing(errorMsg, errorDescr); + THROW_LAST_SYS_ERROR("mkdir"); + } } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(dirPath)), e.toString()); } } -bool zen::createDirectoryIfMissingRecursion(const Zstring& dirPath) //throw FileError +void zen::createDirectoryIfMissingRecursion(const Zstring& dirPath) //throw FileError { const std::optional<Zstring> parentPath = getParentFolderPath(dirPath); if (!parentPath) //device root - return false; + return; - try //generally we expect that path already exists (see: ffs_paths.cpp) => check first + try //generally expect folder already exists (see: ffs_paths.cpp) => check first { if (getItemType(dirPath) != ItemType::file) //throw FileError - return false; + return; } catch (FileError&) {} //not yet existing or access error? let's find out... @@ -511,14 +510,14 @@ bool zen::createDirectoryIfMissingRecursion(const Zstring& dirPath) //throw File try { createDirectory(dirPath); //throw FileError, ErrorTargetExisting - return true; + return; } catch (FileError&) { try { if (getItemType(dirPath) != ItemType::file) //throw FileError - return true; //already existing => possible, if createDirectoryIfMissingRecursion() is run in parallel + return; //already existing => possible, if createDirectoryIfMissingRecursion() is run in parallel } catch (FileError&) {} //not yet existing or access error @@ -529,6 +528,11 @@ bool zen::createDirectoryIfMissingRecursion(const Zstring& dirPath) //throw File void zen::tryCopyDirectoryAttributes(const Zstring& sourcePath, const Zstring& targetPath) //throw FileError { +//do NOT copy attributes for volume root paths which return as: FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY +//https://freefilesync.org/forum/viewtopic.php?t=5550 +if (!getParentFolderPath(sourcePath)) //=> root path + return; + } @@ -549,6 +553,7 @@ void zen::copySymlink(const Zstring& sourcePath, const Zstring& targetPath) //th //allow only consistent objects to be created -> don't place before ::symlink(); targetPath may already exist! ZEN_ON_SCOPE_FAIL(try { removeSymlinkPlain(targetPath); /*throw FileError*/ } catch (FileError&) {}); + warn_static("log it!") //file times: essential for syncing a symlink: enforce this! (don't just try!) struct stat sourceInfo = {}; @@ -562,16 +567,17 @@ void zen::copySymlink(const Zstring& sourcePath, const Zstring& targetPath) //th FileCopyResult zen::copyNewFile(const Zstring& sourceFile, const Zstring& targetFile, //throw FileError, ErrorTargetExisting, (ErrorFileLocked), X const IoCallback& notifyUnbufferedIO /*throw X*/) { - int64_t totalUnbufferedIO = 0; + int64_t totalBytesNotified = 0; + IOCallbackDivider notifyIoDiv(notifyUnbufferedIO, totalBytesNotified); - FileInput fileIn(sourceFile, IOCallbackDivider(notifyUnbufferedIO, totalUnbufferedIO)); //throw FileError, (ErrorFileLocked -> Windows-only) + FileInputPlain fileIn(sourceFile); //throw FileError, (ErrorFileLocked -> Windows-only) - struct stat sourceInfo = {}; - if (::fstat(fileIn.getHandle(), &sourceInfo) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(sourceFile)), "fstat"); + const struct stat& sourceInfo = fileIn.getStatBuffered(); //throw FileError - const mode_t mode = sourceInfo.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); //analog to "cp" which copies "mode" (considering umask) by default - //it seems we don't need S_IWUSR, not even for the setFileTime() below! (tested with source file having different user/group!) + //analog to "cp" which copies "mode" (considering umask) by default: + const mode_t mode = (sourceInfo.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) | + S_IWUSR;//macOS: S_IWUSR apparently needed to write extended attributes (see copyfile() function) + //Linux: not needed even for the setFileTime() below! (tested with source file having different user/group!) //=> need copyItemPermissions() only for "chown" and umask-agnostic permissions const int fdTarget = ::open(targetFile.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, mode); @@ -586,27 +592,46 @@ FileCopyResult zen::copyNewFile(const Zstring& sourceFile, const Zstring& target throw FileError(errorMsg, errorDescr); } - FileOutput fileOut(fdTarget, targetFile, IOCallbackDivider(notifyUnbufferedIO, totalUnbufferedIO)); //pass ownership + FileOutputPlain fileOut(fdTarget, targetFile); //pass ownership //preallocate disk space + reduce fragmentation (perf: no real benefit) fileOut.reserveSpace(sourceInfo.st_size); //throw FileError - bufferedStreamCopy(fileIn, fileOut); //throw FileError, (ErrorFileLocked), X + unbufferedStreamCopy([&](void* buffer, size_t bytesToRead) + { + const size_t bytesRead = fileIn.tryRead(buffer, bytesToRead); //throw FileError, (ErrorFileLocked) + notifyIoDiv(bytesRead); //throw X + return bytesRead; + }, + fileIn.getBlockSize() /*throw FileError*/, + + [&](const void* buffer, size_t bytesToWrite) + { + const size_t bytesWritten = fileOut.tryWrite(buffer, bytesToWrite); //throw FileError + notifyIoDiv(bytesWritten); //throw X + return bytesWritten; + }, + fileOut.getBlockSize() /*throw FileError*/); //throw FileError, X + +#if 0 + //clean file system cache: needed at all? no user complaints at all!!! + //posix_fadvise(POSIX_FADV_DONTNEED) does nothing, unless data was already read from/written to disk: https://insights.oetiker.ch/linux/fadvise/ + // => should be "most" of the data at this point => good enough? + if (::posix_fadvise(fileIn.getHandle(), 0 /*offset*/, 0 /*len*/, POSIX_FADV_DONTNEED) != 0) //"len == 0" means "end of the file" + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(sourceFile)), "posix_fadvise(POSIX_FADV_DONTNEED)"); + if (::posix_fadvise(fileOut.getHandle(), 0 /*offset*/, 0 /*len*/, POSIX_FADV_DONTNEED) != 0) //"len == 0" means "end of the file" + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(targetFile)), "posix_fadvise(POSIX_FADV_DONTNEED)"); +#endif - //flush intermediate buffers before fiddling with the raw file handle - fileOut.flushBuffers(); //throw FileError, X - struct stat targetInfo = {}; - if (::fstat(fileOut.getHandle(), &targetInfo) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(targetFile)), "fstat"); + const auto targetFileIdx = fileOut.getStatBuffered().st_ino; //throw FileError //close output file handle before setting file time; also good place to catch errors when closing stream! - fileOut.finalize(); //throw FileError, (X) essentially a close() since buffers were already flushed - + fileOut.close(); //throw FileError //========================================================================================================== - //take fileOut ownership => from this point on, WE are responsible for calling removeFilePlain() on failure!! + //take over fileOut ownership => from this point on, WE are responsible for calling removeFilePlain() on failure!! + // not needed *currently*! see below: ZEN_ON_SCOPE_FAIL(try { removeFilePlain(targetFile); } catch (FileError&) {}); //=========================================================================================================== - std::optional<FileError> errorModTime; try { @@ -622,13 +647,14 @@ FileCopyResult zen::copyNewFile(const Zstring& sourceFile, const Zstring& target errorModTime = FileError(e.toString()); //avoid slicing } - FileCopyResult result; - result.fileSize = sourceInfo.st_size; - result.sourceModTime = sourceInfo.st_mtim; - result.sourceFileIdx = sourceInfo.st_ino; - result.targetFileIdx = targetInfo.st_ino; - result.errorModTime = errorModTime; - return result; + return + { + .fileSize = makeUnsigned(sourceInfo.st_size), + .sourceModTime = sourceInfo.st_mtim, + .sourceFileIdx = sourceInfo.st_ino, + .targetFileIdx = targetFileIdx, + .errorModTime = errorModTime, + }; } diff --git a/zen/file_access.h b/zen/file_access.h index f6a02edc..639abf64 100644 --- a/zen/file_access.h +++ b/zen/file_access.h @@ -72,8 +72,7 @@ void copyItemPermissions(const Zstring& sourcePath, const Zstring& targetPath, P void createDirectory(const Zstring& dirPath); //throw FileError, ErrorTargetExisting //creates directories recursively if not existing -//returns false if folder already exists -bool createDirectoryIfMissingRecursion(const Zstring& dirPath); //throw FileError +void createDirectoryIfMissingRecursion(const Zstring& dirPath); //throw FileError //symlink handling: follow //expects existing source/target directories diff --git a/zen/file_error.h b/zen/file_error.h index 168ea806..93c95f90 100644 --- a/zen/file_error.h +++ b/zen/file_error.h @@ -30,6 +30,7 @@ private: DEFINE_NEW_FILE_ERROR(ErrorTargetExisting) DEFINE_NEW_FILE_ERROR(ErrorFileLocked) DEFINE_NEW_FILE_ERROR(ErrorMoveUnsupported) +DEFINE_NEW_FILE_ERROR(RecycleBinUnavailable) //CAVEAT: thread-local Win32 error code is easily overwritten => evaluate *before* making any (indirect) system calls: diff --git a/zen/file_io.cpp b/zen/file_io.cpp index 7eebd2e8..ef3cbebb 100644 --- a/zen/file_io.cpp +++ b/zen/file_io.cpp @@ -5,7 +5,6 @@ // ***************************************************************************** #include "file_io.h" - #include <sys/stat.h> #include <fcntl.h> //open #include <unistd.h> //close, read, write @@ -13,6 +12,45 @@ using namespace zen; +size_t FileBase::getBlockSize() //throw FileError +{ + if (blockSizeBuf_ == 0) + { + /* - statfs::f_bsize - "optimal transfer block size" + - stat::st_blksize - "blocksize for file system I/O. Writing in smaller chunks may cause an inefficient read-modify-rewrite." + + e.g. local disk: f_bsize 4096 st_blksize 4096 + USB memory: f_bsize 32768 st_blksize 32768 */ + const auto st_blksize = getStatBuffered().st_blksize; //throw FileError + if (st_blksize > 0) //st_blksize is signed! + blockSizeBuf_ = st_blksize; // + + blockSizeBuf_ = std::max(blockSizeBuf_, defaultBlockSize); + //ha, convergent evolution! https://github.com/coreutils/coreutils/blob/master/src/ioblksize.h#L74 + } + return blockSizeBuf_; +} + + +const struct stat& FileBase::getStatBuffered() //throw FileError +{ + if (!statBuf_) + try + { + if (hFile_ == invalidFileHandle) + throw SysError(L"Contract error: getStatBuffered() called after close()."); + + struct stat fileInfo = {}; + if (::fstat(hFile_, &fileInfo) != 0) + THROW_LAST_SYS_ERROR("fstat"); + statBuf_ = std::move(fileInfo); + } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath_)), e.toString()); } + + return *statBuf_; +} + + FileBase::~FileBase() { if (hFile_ != invalidFileHandle) @@ -21,29 +59,37 @@ FileBase::~FileBase() close(); //throw FileError } catch (FileError&) { assert(false); } + warn_static("log it!") } void FileBase::close() //throw FileError { - if (hFile_ == invalidFileHandle) - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"Contract error: close() called more than once."); - ZEN_ON_SCOPE_EXIT(hFile_ = invalidFileHandle); - - if (::close(hFile_) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), "close"); + try + { + if (hFile_ == invalidFileHandle) + throw SysError(L"Contract error: close() called more than once."); + if (::close(hFile_) != 0) + THROW_LAST_SYS_ERROR("close"); + hFile_ = invalidFileHandle; //do NOT set on failure! => ~FileOutputPlain() still wants to (try to) delete the file! + } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), e.toString()); } } //---------------------------------------------------------------------------------------------------- namespace { -FileBase::FileHandle openHandleForRead(const Zstring& filePath) //throw FileError, ErrorFileLocked + std::pair<FileBase::FileHandle, struct stat> +openHandleForRead(const Zstring& filePath) //throw FileError, ErrorFileLocked { - //caveat: check for file types that block during open(): character device, block device, named pipe - struct stat fileInfo = {}; - if (::stat(filePath.c_str(), &fileInfo) == 0) //follows symlinks + try { + //caveat: check for file types that block during open(): character device, block device, named pipe + struct stat fileInfo = {}; + if (::stat(filePath.c_str(), &fileInfo) != 0) //follows symlinks + THROW_LAST_SYS_ERROR("stat"); + if (!S_ISREG(fileInfo.st_mode) && !S_ISDIR(fileInfo.st_mode) && //open() will fail with "EISDIR: Is a directory" => nice !S_ISLNK(fileInfo.st_mode)) //?? shouldn't be possible after successful stat() @@ -59,103 +105,77 @@ FileBase::FileHandle openHandleForRead(const Zstring& filePath) //throw FileErro name += L", "; return name + printNumber<std::wstring>(L"0%06o", m & S_IFMT); }(); - throw FileError(replaceCpy(_("Cannot open file %x."), L"%x", fmtPath(filePath)), - _("Unsupported item type.") + L" [" + typeName + L']'); + throw SysError(_("Unsupported item type.") + L" [" + typeName + L']'); } - } - //else: let ::open() fail for errors like "not existing" - //don't use O_DIRECT: https://yarchive.net/comp/linux/o_direct.html - const int fdFile = ::open(filePath.c_str(), O_RDONLY | O_CLOEXEC); - if (fdFile == -1) //don't check "< 0" -> docu seems to allow "-2" to be a valid file handle - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot open file %x."), L"%x", fmtPath(filePath)), "open"); - return fdFile; //pass ownership + //don't use O_DIRECT: https://yarchive.net/comp/linux/o_direct.html + const int fdFile = ::open(filePath.c_str(), O_RDONLY | O_CLOEXEC); + if (fdFile == -1) //don't check "< 0" -> docu seems to allow "-2" to be a valid file handle + THROW_LAST_SYS_ERROR("open"); + return {fdFile /*pass ownership*/, fileInfo}; + } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot open file %x."), L"%x", fmtPath(filePath)), e.toString()); } } } -FileInput::FileInput(FileHandle handle, const Zstring& filePath, const IoCallback& notifyUnbufferedIO) : - FileBase(handle, filePath), notifyUnbufferedIO_(notifyUnbufferedIO) {} +FileInputPlain::FileInputPlain(const Zstring& filePath) : + FileInputPlain(openHandleForRead(filePath), filePath) {} //throw FileError, ErrorFileLocked + + +FileInputPlain::FileInputPlain(const std::pair<FileBase::FileHandle, struct stat>& fileDetails, const Zstring& filePath) : + FileInputPlain(fileDetails.first, filePath) +{ + setStatBuffered(fileDetails.second); +} -FileInput::FileInput(const Zstring& filePath, const IoCallback& notifyUnbufferedIO) : - FileBase(openHandleForRead(filePath), filePath), //throw FileError, ErrorFileLocked - notifyUnbufferedIO_(notifyUnbufferedIO) +FileInputPlain::FileInputPlain(FileHandle handle, const Zstring& filePath) : + FileBase(handle, filePath) { //optimize read-ahead on input file: - if (::posix_fadvise(getHandle(), 0, 0, POSIX_FADV_SEQUENTIAL) != 0) + if (::posix_fadvise(getHandle(), 0 /*offset*/, 0 /*len*/, POSIX_FADV_SEQUENTIAL) != 0) //"len == 0" means "end of the file" THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(filePath)), "posix_fadvise(POSIX_FADV_SEQUENTIAL)"); + /* - POSIX_FADV_SEQUENTIAL is like POSIX_FADV_NORMAL, but with twice the read-ahead buffer size + - POSIX_FADV_NOREUSE "since kernel 2.6.18 this flag is a no-op" WTF!? + - POSIX_FADV_DONTNEED may be used to clear the OS file system cache (offset and len must be page-aligned!) + => does nothing, unless data was already written to disk: https://insights.oetiker.ch/linux/fadvise/ + - POSIX_FADV_WILLNEED: issue explicit read-ahead; almost the same as readahead(), but with weaker error checking + https://unix.stackexchange.com/questions/681188/difference-between-posix-fadvise-and-readahead + + clear file system cache manually: sync; echo 3 > /proc/sys/vm/drop_caches */ + } -size_t FileInput::tryRead(void* buffer, size_t bytesToRead) //throw FileError, ErrorFileLocked; may return short, only 0 means EOF! +//may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! +size_t FileInputPlain::tryRead(void* buffer, size_t bytesToRead) //throw FileError, ErrorFileLocked { if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - assert(bytesToRead == getBlockSize()); - - ssize_t bytesRead = 0; - do + assert(bytesToRead % getBlockSize() == 0); + try { - bytesRead = ::read(getHandle(), buffer, bytesToRead); - } - while (bytesRead < 0 && errno == EINTR); //Compare copy_reg() in copy.c: ftp://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz - //EINTR is not checked on macOS' copyfile: https://opensource.apple.com/source/copyfile/copyfile-146/copyfile.c.auto.html - //read() on macOS: https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man2/read.2.html - - if (bytesRead < 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), "read"); - if (static_cast<size_t>(bytesRead) > bytesToRead) //better safe than sorry - throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), formatSystemError("ReadFile", L"", L"Buffer overflow.")); - - //if ::read is interrupted (EINTR) right in the middle, it will return successfully with "bytesRead < bytesToRead" + ssize_t bytesRead = 0; + do + { + bytesRead = ::read(getHandle(), buffer, bytesToRead); + } + while (bytesRead < 0 && errno == EINTR); //Compare copy_reg() in copy.c: ftp://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz + //EINTR is not checked on macOS' copyfile: https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html + //read() on macOS: https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man2/read.2.html - return bytesRead; //"zero indicates end of file" -} + if (bytesRead < 0) + THROW_LAST_SYS_ERROR("read"); + if (makeUnsigned(bytesRead) > bytesToRead) //better safe than sorry + throw SysError(formatSystemError("ReadFile", L"", L"Buffer overflow.")); + //if ::read is interrupted (EINTR) right in the middle, it will return successfully with "bytesRead < bytesToRead" -size_t FileInput::read(void* buffer, size_t bytesToRead) //throw FileError, ErrorFileLocked, X; return "bytesToRead" bytes unless end of stream! -{ - /* - FFS 8.9-9.5 perf issues on macOS: https://freefilesync.org/forum/viewtopic.php?t=4808 - app-level buffering is essential to optimize random data sizes; e.g. "export file list": - => big perf improvement on Windows, Linux. No significant improvement on macOS in tests - impact on stream-based file copy: - => no drawback vs block-wise copy loop on Linux, HOWEVER: big perf issue on macOS! - - Possible cause of macOS perf issue unclear: - - getting rid of std::vector::resize() and std::vector::erase() "fixed" the problem - => costly zero-initializing memory? problem with inlining? QOI issue of std:vector on clang/macOS? - - replacing std::copy() with memcpy() also *seems* to have improved speed "somewhat" - */ - - const size_t blockSize = getBlockSize(); - assert(memBuf_.size() >= blockSize); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); - - auto it = static_cast<std::byte*>(buffer); - const auto itEnd = it + bytesToRead; - for (;;) - { - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), bufPosEnd_ - bufPos_); - std::memcpy(it, &memBuf_[0] + bufPos_ /*caveat: vector debug checks*/, junkSize); - bufPos_ += junkSize; - it += junkSize; - - if (it == itEnd) - break; - //-------------------------------------------------------------------- - const size_t bytesRead = tryRead(&memBuf_[0], blockSize); //throw FileError, ErrorFileLocked; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 - bufPos_ = 0; - bufPosEnd_ = bytesRead; - - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesRead); //throw X - - if (bytesRead == 0) //end of file - break; + return bytesRead; //"zero indicates end of file" } - return it - static_cast<std::byte*>(buffer); + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), e.toString()); } } //---------------------------------------------------------------------------------------------------- @@ -164,170 +184,153 @@ namespace { FileBase::FileHandle openHandleForWrite(const Zstring& filePath) //throw FileError, ErrorTargetExisting { - //checkForUnsupportedType(filePath); -> not needed, open() + O_WRONLY should fail fast - - const mode_t lockFileMode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; //0666 => umask will be applied implicitly! - - //O_EXCL contains a race condition on NFS file systems: https://linux.die.net/man/2/open - const int fdFile = ::open(filePath.c_str(), //const char* pathname - O_CREAT | //int flags - /*access == FileOutput::ACC_OVERWRITE ? O_TRUNC : */ O_EXCL | O_WRONLY | O_CLOEXEC, - lockFileMode); //mode_t mode - if (fdFile == -1) + try { - const int ec = errno; //copy before making other system calls! - const std::wstring errorMsg = replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(filePath)); - const std::wstring errorDescr = formatSystemError("open", ec); + //checkForUnsupportedType(filePath); -> not needed, open() + O_WRONLY should fail fast + + const mode_t lockFileMode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; //0666 => umask will be applied implicitly! - if (ec == EEXIST) - throw ErrorTargetExisting(errorMsg, errorDescr); - //if (ec == ENOENT) throw ErrorTargetPathMissing(errorMsg, errorDescr); + //O_EXCL contains a race condition on NFS file systems: https://linux.die.net/man/2/open + const int fdFile = ::open(filePath.c_str(), //const char* pathname + O_CREAT | //int flags + /*access == FileOutput::ACC_OVERWRITE ? O_TRUNC : */ O_EXCL | O_WRONLY | O_CLOEXEC, + lockFileMode); //mode_t mode + if (fdFile == -1) + { + const int ec = errno; //copy before making other system calls! + if (ec == EEXIST) + throw ErrorTargetExisting(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(filePath)), formatSystemError("open", ec)); + //if (ec == ENOENT) throw ErrorTargetPathMissing(errorMsg, errorDescr); - throw FileError(errorMsg, errorDescr); + THROW_LAST_SYS_ERROR("open"); + } + return fdFile; //pass ownership } - return fdFile; //pass ownership + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(filePath)), e.toString()); } } } -FileOutput::FileOutput(FileHandle handle, const Zstring& filePath, const IoCallback& notifyUnbufferedIO) : - FileBase(handle, filePath), notifyUnbufferedIO_(notifyUnbufferedIO) -{ -} +FileOutputPlain::FileOutputPlain(const Zstring& filePath) : + FileOutputPlain(openHandleForWrite(filePath), filePath) {} //throw FileError, ErrorTargetExisting -FileOutput::FileOutput(const Zstring& filePath, const IoCallback& notifyUnbufferedIO) : - FileBase(openHandleForWrite(filePath), filePath), notifyUnbufferedIO_(notifyUnbufferedIO) {} //throw FileError, ErrorTargetExisting +FileOutputPlain::FileOutputPlain(FileHandle handle, const Zstring& filePath) : + FileBase(handle, filePath) +{ +} -FileOutput::~FileOutput() +FileOutputPlain::~FileOutputPlain() { if (getHandle() != invalidFileHandle) //not finalized => clean up garbage - { - //"deleting while handle is open" == FILE_FLAG_DELETE_ON_CLOSE - if (::unlink(getFilePath().c_str()) != 0) + try + { + //"deleting while handle is open" == FILE_FLAG_DELETE_ON_CLOSE + if (::unlink(getFilePath().c_str()) != 0) + THROW_LAST_SYS_ERROR("unlink"); + } + catch (const SysError&) + { assert(false); - } + warn_static("at least log on failure!") + } } -size_t FileOutput::tryWrite(const void* buffer, size_t bytesToWrite) //throw FileError; may return short! CONTRACT: bytesToWrite > 0 +void FileOutputPlain::reserveSpace(uint64_t expectedSize) //throw FileError { - if (bytesToWrite == 0) - throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - assert(bytesToWrite <= getBlockSize()); - - ssize_t bytesWritten = 0; - do - { - bytesWritten = ::write(getHandle(), buffer, bytesToWrite); - } - while (bytesWritten < 0 && errno == EINTR); - //write() on macOS: https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man2/write.2.html + //NTFS: "If you set the file allocation info [...] the file contents will be forced into nonresident data, even if it would have fit inside the MFT." + if (expectedSize < 1024) //https://docs.microsoft.com/en-us/archive/blogs/askcore/the-four-stages-of-ntfs-file-growth + return; - if (bytesWritten <= 0) + try { - if (bytesWritten == 0) //comment in safe-read.c suggests to treat this as an error due to buggy drivers - errno = ENOSPC; + //don't use ::posix_fallocate which uses horribly inefficient fallback if FS doesn't support it (EOPNOTSUPP) and changes files size! + //FALLOC_FL_KEEP_SIZE => allocate only, file size is NOT changed! + if (::fallocate(getHandle(), //int fd + FALLOC_FL_KEEP_SIZE, //int mode + 0, //off_t offset + expectedSize) != 0) //off_t len + if (errno != EOPNOTSUPP) //possible, unlike with posix_fallocate() + THROW_LAST_SYS_ERROR("fallocate"); - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), "write"); } - if (bytesWritten > static_cast<ssize_t>(bytesToWrite)) //better safe than sorry - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), formatSystemError("write", L"", L"Buffer overflow.")); - - //if ::write() is interrupted (EINTR) right in the middle, it will return successfully with "bytesWritten < bytesToWrite"! - return bytesWritten; + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), e.toString()); } } -void FileOutput::write(const void* buffer, size_t bytesToWrite) //throw FileError, X +//may return short! CONTRACT: bytesToWrite > 0 +size_t FileOutputPlain::tryWrite(const void* buffer, size_t bytesToWrite) //throw FileError { - const size_t blockSize = getBlockSize(); - assert(memBuf_.size() >= blockSize); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); - - auto it = static_cast<const std::byte*>(buffer); - const auto itEnd = it + bytesToWrite; - for (;;) + if (bytesToWrite == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + assert(bytesToWrite % getBlockSize() == 0 || bytesToWrite < getBlockSize()); + try { - if (memBuf_.size() - bufPos_ < blockSize) //support memBuf_.size() > blockSize to reduce memmove()s, but perf test shows: not really needed! - // || bufPos_ == bufPosEnd_) -> not needed while memBuf_.size() == blockSize + ssize_t bytesWritten = 0; + do { - std::memmove(&memBuf_[0], &memBuf_[0] + bufPos_, bufPosEnd_ - bufPos_); - bufPosEnd_ -= bufPos_; - bufPos_ = 0; + bytesWritten = ::write(getHandle(), buffer, bytesToWrite); } + while (bytesWritten < 0 && errno == EINTR); + //write() on macOS: https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man2/write.2.html - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), blockSize - (bufPosEnd_ - bufPos_)); - std::memcpy(&memBuf_[0] + bufPosEnd_ /*caveat: vector debug checks*/, it, junkSize); - bufPosEnd_ += junkSize; - it += junkSize; - - if (it == itEnd) - return; - //-------------------------------------------------------------------- - const size_t bytesWritten = tryWrite(&memBuf_[bufPos_], blockSize); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - bufPos_ += bytesWritten; - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesWritten); //throw X! - } -} + if (bytesWritten <= 0) + { + if (bytesWritten == 0) //comment in safe-read.c suggests to treat this as an error due to buggy drivers + errno = ENOSPC; + THROW_LAST_SYS_ERROR("write"); + } + if (bytesWritten > static_cast<ssize_t>(bytesToWrite)) //better safe than sorry + throw SysError(formatSystemError("write", L"", L"Buffer overflow.")); -void FileOutput::flushBuffers() //throw FileError, X -{ - assert(bufPosEnd_ - bufPos_ <= getBlockSize()); - assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); - while (bufPos_ != bufPosEnd_) - { - const size_t bytesWritten = tryWrite(&memBuf_[bufPos_], bufPosEnd_ - bufPos_); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - bufPos_ += bytesWritten; - if (notifyUnbufferedIO_) notifyUnbufferedIO_(bytesWritten); //throw X! + //if ::write() is interrupted (EINTR) right in the middle, it will return successfully with "bytesWritten < bytesToWrite"! + return bytesWritten; } + catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), e.toString()); } } +//---------------------------------------------------------------------------------------------------- -void FileOutput::finalize() //throw FileError, X -{ - flushBuffers(); //throw FileError, X - close(); //throw FileError - //~FileBase() calls this one, too, but we want to propagate errors if any -} - - -void FileOutput::reserveSpace(uint64_t expectedSize) //throw FileError +std::string zen::getFileContent(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { - //NTFS: "If you set the file allocation info [...] the file contents will be forced into nonresident data, even if it would have fit inside the MFT." - if (expectedSize < 1024) //https://www.sciencedirect.com/topics/computer-science/master-file-table - return; - - //don't use ::posix_fallocate which uses horribly inefficient fallback if FS doesn't support it (EOPNOTSUPP) and changes files size! - //FALLOC_FL_KEEP_SIZE => allocate only, file size is NOT changed! - if (::fallocate(getHandle(), //int fd - FALLOC_FL_KEEP_SIZE, //int mode - 0, //off_t offset - expectedSize) != 0) //off_t len - if (errno != EOPNOTSUPP) //possible, unlike with posix_fallocate() - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), "fallocate"); + FileInputPlain fileIn(filePath); //throw FileError, ErrorFileLocked + return unbufferedLoad<std::string>([&](void* buffer, size_t bytesToRead) + { + const size_t bytesRead = fileIn.tryRead(buffer, bytesToRead); //throw FileError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesRead); //throw X! + return bytesRead; + }, + fileIn.getBlockSize()); //throw FileError, X } -std::string zen::getFileContent(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X +void zen::setFileContent(const Zstring& filePath, const std::string& byteStream, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { - FileInput streamIn(filePath, notifyUnbufferedIO); //throw FileError, ErrorFileLocked - return bufferedLoad<std::string>(streamIn); //throw FileError, X -} + const Zstring tmpFilePath = getPathWithTempName(filePath); + FileOutputPlain tmpFile(tmpFilePath); //throw FileError, (ErrorTargetExisting) -void zen::setFileContent(const Zstring& filePath, const std::string& byteStream, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X -{ - TempFileOutput fileOut(filePath, notifyUnbufferedIO); //throw FileError - if (!byteStream.empty()) + tmpFile.reserveSpace(byteStream.size()); //throw FileError + + unbufferedSave(byteStream, [&](const void* buffer, size_t bytesToWrite) { - //preallocate disk space & reduce fragmentation - fileOut.reserveSpace(byteStream.size()); //throw FileError - fileOut.write(&byteStream[0], byteStream.size()); //throw FileError, X - } - fileOut.commit(); //throw FileError, X + const size_t bytesWritten = tmpFile.tryWrite(buffer, bytesToWrite); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesWritten); //throw X! + return bytesWritten; + }, + tmpFile.getBlockSize()); //throw FileError, X + + tmpFile.close(); //throw FileError + //take over ownership: + ZEN_ON_SCOPE_FAIL( try { removeFilePlain(tmpFilePath); /*throw FileError*/ } + catch (FileError&) {}); + warn_static("log it!") + + //operation finished: move temp file transactionally + moveAndRenameItem(tmpFilePath, filePath, true /*replaceExisting*/); //throw FileError, (ErrorMoveUnsupported), (ErrorTargetExisting) } diff --git a/zen/file_io.h b/zen/file_io.h index f1e4200d..46ffa843 100644 --- a/zen/file_io.h +++ b/zen/file_io.h @@ -8,7 +8,7 @@ #define FILE_IO_H_89578342758342572345 #include "file_access.h" -//#include "serialize.h" +#include "serialize.h" #include "crc.h" #include "guid.h" @@ -17,7 +17,7 @@ namespace zen { const char LINE_BREAK[] = "\n"; //since OS X Apple uses newline, too -/* OS-buffered file IO optimized for +/* OS-buffered file I/O: - sequential read/write accesses - better error reporting - long path support @@ -30,17 +30,21 @@ public: FileHandle getHandle() { return hFile_; } - //Windows: use 64kB ?? https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-2000-server/cc938632%28v=technet.10%29 - //macOS, Linux: use st_blksize? - static size_t getBlockSize() { return 128 * 1024; }; - const Zstring& getFilePath() const { return filePath_; } + size_t getBlockSize(); //throw FileError + + static constexpr size_t defaultBlockSize = 256 * 1024; + + void close(); //throw FileError -> good place to catch errors when closing stream, otherwise called in ~FileBase()! + + const struct stat& getStatBuffered(); //throw FileError + protected: FileBase(FileHandle handle, const Zstring& filePath) : hFile_(handle), filePath_(filePath) {} ~FileBase(); - void close(); //throw FileError -> optional, but good place to catch errors when closing stream! + void setStatBuffered(const struct stat& fileInfo) { statBuf_ = fileInfo; } private: FileBase (const FileBase&) = delete; @@ -48,88 +52,125 @@ private: FileHandle hFile_ = invalidFileHandle; const Zstring filePath_; + size_t blockSizeBuf_ = 0; + std::optional<struct stat> statBuf_; }; //----------------------------------------------------------------------------------------------- -class FileInput : public FileBase +class FileInputPlain : public FileBase { public: - FileInput( const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, ErrorFileLocked - FileInput(FileHandle handle, const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/); //takes ownership! + FileInputPlain( const Zstring& filePath); //throw FileError, ErrorFileLocked + FileInputPlain(FileHandle handle, const Zstring& filePath); //takes ownership! - size_t read(void* buffer, size_t bytesToRead); //throw FileError, ErrorFileLocked, X; return "bytesToRead" bytes unless end of stream! + //may return short, only 0 means EOF! CONTRACT: bytesToRead > 0! + size_t tryRead(void* buffer, size_t bytesToRead); //throw FileError, ErrorFileLocked private: - size_t tryRead(void* buffer, size_t bytesToRead); //throw FileError, ErrorFileLocked; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! - - const IoCallback notifyUnbufferedIO_; //throw X - - std::vector<std::byte> memBuf_ = std::vector<std::byte>(getBlockSize()); - size_t bufPos_ = 0; - size_t bufPosEnd_= 0; + FileInputPlain(const std::pair<FileBase::FileHandle, struct stat>& fileDetails, const Zstring& filePath); }; -class FileOutput : public FileBase +class FileOutputPlain : public FileBase { public: - FileOutput( const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, ErrorTargetExisting - FileOutput(FileHandle handle, const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/); //takes ownership! - ~FileOutput(); + FileOutputPlain( const Zstring& filePath); //throw FileError, ErrorTargetExisting + FileOutputPlain(FileHandle handle, const Zstring& filePath); //takes ownership! + ~FileOutputPlain(); + //preallocate disk space & reduce fragmentation void reserveSpace(uint64_t expectedSize); //throw FileError - void write(const void* buffer, size_t bytesToWrite); //throw FileError, X - void flushBuffers(); //throw FileError, X - //caveat: does NOT flush OS or hard disk buffers like e.g. FlushFileBuffers()! + //may return short! CONTRACT: bytesToWrite > 0 + size_t tryWrite(const void* buffer, size_t bytesToWrite); //throw FileError - void finalize(); /*= flushBuffers() + close()*/ //throw FileError, X + //close() when done, or else file is considered incomplete and will be deleted! private: - size_t tryWrite(const void* buffer, size_t bytesToWrite); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 - - IoCallback notifyUnbufferedIO_; //throw X - std::vector<std::byte> memBuf_ = std::vector<std::byte>(getBlockSize()); - size_t bufPos_ = 0; - size_t bufPosEnd_ = 0; }; -//----------------------------------------------------------------------------------------------- -//native stream I/O convenience functions: -class TempFileOutput +//-------------------------------------------------------------------- + +namespace impl +{ +inline +auto makeTryRead(FileInputPlain& fip, const IoCallback& notifyUnbufferedIO /*throw X*/) +{ + return [&](void* buffer, size_t bytesToRead) + { + const size_t bytesRead = fip.tryRead(buffer, bytesToRead); //throw FileError, ErrorFileLocked; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesRead); //throw X + return bytesRead; + }; +} + + +inline +auto makeTryWrite(FileOutputPlain& fop, const IoCallback& notifyUnbufferedIO /*throw X*/) +{ + return [&](const void* buffer, size_t bytesToWrite) + { + const size_t bytesWritten = fop.tryWrite(buffer, bytesToWrite); //throw FileError + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesWritten); //throw X + return bytesWritten; + }; +} +} + +//-------------------------------------------------------------------- + +class FileInputBuffered { public: - TempFileOutput( const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : //throw FileError - filePath_(filePath), - tmpFile_(tmpFilePath_, notifyUnbufferedIO) {} //throw FileError, (ErrorTargetExisting) + FileInputBuffered(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : //throw FileError, ErrorFileLocked + fileIn_(filePath), //throw FileError, ErrorFileLocked + notifyUnbufferedIO_(notifyUnbufferedIO) {} - void reserveSpace(uint64_t expectedSize) { tmpFile_.reserveSpace(expectedSize); } //throw FileError + //return "bytesToRead" bytes unless end of stream! + size_t read(void* buffer, size_t bytesToRead) { return streamIn_.read(buffer, bytesToRead); } //throw FileError, ErrorFileLocked, X + +private: + FileInputPlain fileIn_; + const IoCallback notifyUnbufferedIO_; //throw X - void write(const void* buffer, size_t bytesToWrite) { tmpFile_.write(buffer, bytesToWrite); } //throw FileError, X + BufferedInputStream<FunctionReturnTypeT<decltype(&impl::makeTryRead)>> + streamIn_{impl::makeTryRead(fileIn_, notifyUnbufferedIO_), fileIn_.getBlockSize()}; //throw FileError +}; - FileOutput& refTempFile() { return tmpFile_; } - void commit() //throw FileError, X - { - tmpFile_.finalize(); //throw FileError, X +class FileOutputBuffered +{ +public: + FileOutputBuffered(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/) : //throw FileError, ErrorTargetExisting + fileOut_(filePath), //throw FileError, ErrorTargetExisting + notifyUnbufferedIO_(notifyUnbufferedIO) {} - //take ownership: - ZEN_ON_SCOPE_FAIL( try { removeFilePlain(tmpFilePath_); /*throw FileError*/ } - catch (FileError&) {}); + void write(const void* buffer, size_t bytesToWrite) { streamOut_.write(buffer, bytesToWrite); } //throw FileError, X - //operation finished: move temp file transactionally - moveAndRenameItem(tmpFilePath_, filePath_, true /*replaceExisting*/); //throw FileError, (ErrorMoveUnsupported), (ErrorTargetExisting) + void finalize() //throw FileError, X + { + streamOut_.flushBuffer(); //throw FileError, X + fileOut_.close(); //throw FileError } private: - //generate (hopefully) unique file name to avoid clashing with unrelated tmp file - const Zstring filePath_; - const Zstring shortGuid_ = printNumber<Zstring>(Zstr("%04x"), static_cast<unsigned int>(getCrc16(generateGUID()))); - const Zstring tmpFilePath_ = filePath_ + Zstr('.') + shortGuid_ + Zstr(".tmp"); - FileOutput tmpFile_; + FileOutputPlain fileOut_; + const IoCallback notifyUnbufferedIO_; //throw X + + BufferedOutputStream<FunctionReturnTypeT<decltype(&impl::makeTryWrite)>> + streamOut_{impl::makeTryWrite(fileOut_, notifyUnbufferedIO_), fileOut_.getBlockSize()}; //throw FileError }; +//----------------------------------------------------------------------------------------------- + +//stream I/O convenience functions: +inline +Zstring getPathWithTempName(const Zstring& filePath) //generate (hopefully) unique file name +{ + const Zstring shortGuid_ = printNumber<Zstring>(Zstr("%04x"), static_cast<unsigned int>(getCrc16(generateGUID()))); + return filePath + Zstr('.') + shortGuid_ + Zstr(".tmp"); +} [[nodiscard]] std::string getFileContent(const Zstring& filePath, const IoCallback& notifyUnbufferedIO /*throw X*/); //throw FileError, X diff --git a/zen/file_path.cpp b/zen/file_path.cpp index f5c207f3..912d5a37 100644 --- a/zen/file_path.cpp +++ b/zen/file_path.cpp @@ -70,7 +70,7 @@ std::optional<Zstring> zen::getParentFolderPath(const Zstring& itemPath) return appendPath(pc->rootPath, beforeLast(pc->relPath, FILE_NAME_SEPARATOR, IfNotFoundReturn::none)); } - assert(false); + assert(itemPath.empty()); return std::nullopt; } diff --git a/zen/globals.h b/zen/globals.h index dd0dfed9..9e22f56b 100644 --- a/zen/globals.h +++ b/zen/globals.h @@ -10,7 +10,6 @@ #include <atomic> #include <memory> #include "scope_guard.h" -#include "legacy_compiler.h" namespace zen @@ -188,7 +187,7 @@ void registerGlobalForDestruction(CleanUpEntry& entry) static struct { PodSpinMutex spinLock; - CleanUpEntry* head; + CleanUpEntry* head = nullptr; } cleanUpList; static_assert(std::is_trivially_destructible_v<decltype(cleanUpList)>, "we must not generate code for magic statics!"); @@ -25,7 +25,7 @@ std::string generateGUID() //creates a 16-byte GUID #endif #if __GLIBC_PREREQ(2, 25) //getentropy() requires Glibc 2.25 (ldd --version) PS: CentOS 7 is on 2.17 - if (::getentropy(&guid[0], guid.size()) != 0) //"The maximum permitted value for the length argument is 256" + if (::getentropy(guid.data(), guid.size()) != 0) //"The maximum permitted value for the length argument is 256" throw std::runtime_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Failed to generate GUID." + "\n\n" + utfTo<std::string>(formatSystemError("getentropy", errno))); #else @@ -58,7 +58,7 @@ std::string generateGUID() //creates a 16-byte GUID const int fd_ = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC); }; thread_local RandomGeneratorPosix gen; - gen.getBytes(&guid[0], guid.size()); + gen.getBytes(guid.data(), guid.size()); #endif return guid; diff --git a/zen/http.cpp b/zen/http.cpp index a26bb3a5..ee71e5b3 100644 --- a/zen/http.cpp +++ b/zen/http.cpp @@ -8,12 +8,18 @@ #include <libcurl/curl_wrap.h> //DON'T include <curl/curl.h> directly! #include "stream_buffer.h" - #include "thread.h" using namespace zen; + const int HTTP_ACCESS_TIME_OUT_SEC = 20; +const size_t HTTP_BLOCK_SIZE_DOWNLOAD = 64 * 1024; //libcurl returns blocks of only 16 kB as returned by recv() even if we request larger blocks via CURLOPT_BUFFERSIZE +//- InternetReadFile() is buffered + prefetching +//- libcurl returns blocks of only 16 kB as returned by recv() even if we request larger blocks via CURLOPT_BUFFERSIZE + const size_t HTTP_STREAM_BUFFER_SIZE = 1024 * 1024; //unit: [byte] + //stream buffer should be big enough to facilitate prefetching during alternating read/write operations => e.g. see serialize.h::unbufferedStreamCopy() + @@ -23,16 +29,14 @@ public: Impl(const Zstring& url, const std::string* postBuf, //issue POST if bound, GET otherwise const std::string& contentType, //required for POST + const IoCallback& onPostBytesSent /*throw X*/, bool disableGetCache, //not relevant for POST (= never cached) const Zstring& userAgent, - const Zstring& caCertFilePath, //optional: enable certificate validation - const IoCallback& notifyUnbufferedIO /*throw X*/) : //throw SysError, X - notifyUnbufferedIO_(notifyUnbufferedIO) + const Zstring& caCertFilePath /*optional: enable certificate validation*/) //throw SysError, X { ZEN_ON_SCOPE_FAIL(cleanup()); //destructor call would lead to member double clean-up!!! - //may be sending large POST: call back first - if (notifyUnbufferedIO_) notifyUnbufferedIO_(0); //throw X + assert(postBuf || !onPostBytesSent); const Zstring urlFmt = afterFirst(url, Zstr("://"), IfNotFoundReturn::none); const Zstring server = beforeFirst(urlFmt, Zstr('/'), IfNotFoundReturn::all); @@ -61,12 +65,14 @@ public: auto promiseHeader = std::make_shared<std::promise<std::string>>(); std::future<std::string> futHeader = promiseHeader->get_future(); - worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, promiseHeader, headers = std::move(headers), + auto postBytesSent = std::make_shared<std::atomic<int64_t>>(0); + + worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, promiseHeader, headers = std::move(headers), postBytesSent, server, useTls, caCertFilePath, userAgent = utfTo<std::string>(userAgent), postBuf = postBuf ? std::optional<std::string>(*postBuf) : std::nullopt, //[!] life-time! serverRelPath = utfTo<std::string>(page)] { - setCurrentThreadName(Zstr("HttpInputStream ") + server); + setCurrentThreadName(Zstr("Istream ") + server); bool headerReceived = false; try @@ -77,13 +83,22 @@ public: std::vector<CurlOption> extraOptions {{CURLOPT_USERAGENT, userAgent.c_str()}}; //CURLOPT_FOLLOWLOCATION already off by default :) + + + std::function<size_t(std::span<char> buf)> readRequest; if (postBuf) { - extraOptions.emplace_back(CURLOPT_POSTFIELDS, postBuf->c_str()); - extraOptions.emplace_back(CURLOPT_POSTFIELDSIZE_LARGE, postBuf->size()); //postBuf not necessarily null-terminated! + readRequest = [&, postBufStream{MemoryStreamIn(*postBuf)}](std::span<char> buf) mutable + { + const size_t bytesRead = postBufStream.read(buf.data(), buf.size()); + *postBytesSent += bytesRead; + return bytesRead; + }; + extraOptions.emplace_back(CURLOPT_POST, 1); + extraOptions.emplace_back(CURLOPT_POSTFIELDSIZE_LARGE, postBuf->size()); //avoid HTTP chunked transfer encoding? } - //carefully with these callbacks! First receive HTTP header without blocking, + //careful with these callbacks! First receive HTTP header without blocking, //and only then allow AsyncStreamBuffer::write() which can block! std::string headerBuf; @@ -109,13 +124,13 @@ public: if (!headerReceived) throw SysError(L"Received HTTP body without header."); - return asyncStreamOut->write(buf.data(), buf.size()); //throw ThreadStopRequest + asyncStreamOut->write(buf.data(), buf.size()); //throw ThreadStopRequest }; httpSession.perform(serverRelPath, curlHeaders, extraOptions, writeResponse /*throw ThreadStopRequest*/, - nullptr /*readRequest*/, + readRequest, onHeaderData /*throw SysError*/, HTTP_ACCESS_TIME_OUT_SEC); //throw SysError, ThreadStopRequest @@ -133,6 +148,19 @@ public: } }); + //------------------------------------------------------------------------------------ + if (postBuf && onPostBytesSent) + { + int64_t bytesReported = 0; + while (futHeader.wait_for(std::chrono::milliseconds(50)) == std::future_status::timeout) + { + const int64_t bytesDelta = *postBytesSent /*atomic shared access!*/- bytesReported; + bytesReported += bytesDelta; + onPostBytesSent(bytesDelta); //throw X + } + } + //------------------------------------------------------------------------------------ + const std::string headBuf = futHeader.get(); //throw SysError //parse header: https://www.w3.org/Protocols/HTTP/1.0/spec.html#Request-Line const std::string& statusBuf = beforeFirst(headBuf, "\r\n", IfNotFoundReturn::all); @@ -151,9 +179,6 @@ public: /* let's NOT consider "Content-Length" header: - may be unavailable ("Transfer-Encoding: chunked") - may refer to compressed data size ("Content-Encoding: gzip") */ - - //let's not get too finicky: at least report the logical amount of bytes sent/received (excluding HTTP headers) - if (notifyUnbufferedIO_) notifyUnbufferedIO_(postBuf ? postBuf->size() : 0); //throw X } ~Impl() { cleanup(); } @@ -166,39 +191,28 @@ public: return it != responseHeaders_.end() ? &it->second : nullptr; } - size_t read(void* buffer, size_t bytesToRead) //throw SysError, X; return "bytesToRead" bytes unless end of stream! + size_t getBlockSize() const { return HTTP_BLOCK_SIZE_DOWNLOAD; } + + size_t tryRead(void* buffer, size_t bytesToRead) //throw SysError; may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! { - const size_t bytesRead = asyncStreamIn_->read(buffer, bytesToRead); //throw SysError - reportBytesProcessed(); //throw X - return bytesRead; + return asyncStreamIn_->tryRead(buffer, bytesToRead); //throw SysError //no need for asyncStreamIn_->checkWriteErrors(): once end of stream is reached, asyncStreamOut->closeStream() was called => no errors occured } - size_t getBlockSize() const { return 64 * 1024; } - private: Impl (const Impl&) = delete; Impl& operator=(const Impl&) = delete; - void reportBytesProcessed() //throw X - { - const int64_t totalBytesDownloaded = asyncStreamIn_->getTotalBytesWritten(); - if (notifyUnbufferedIO_) notifyUnbufferedIO_(totalBytesDownloaded - totalBytesReported_); //throw X - totalBytesReported_ = totalBytesDownloaded; - } - void cleanup() { asyncStreamIn_->setReadError(std::make_exception_ptr(ThreadStopRequest())); + warn_static("log on error!") } - std::shared_ptr<AsyncStreamBuffer> asyncStreamIn_ = std::make_shared<AsyncStreamBuffer>(512 * 1024); + std::shared_ptr<AsyncStreamBuffer> asyncStreamIn_ = std::make_shared<AsyncStreamBuffer>(HTTP_STREAM_BUFFER_SIZE); InterruptibleThread worker_; - int64_t totalBytesReported_ = 0; int statusCode_ = 0; std::unordered_map<std::string, std::string, StringHashAsciiNoCase, StringEqualAsciiNoCase> responseHeaders_; - - const IoCallback notifyUnbufferedIO_; //throw X }; @@ -206,11 +220,20 @@ HttpInputStream::HttpInputStream(std::unique_ptr<Impl>&& pimpl) : pimpl_(std::mo HttpInputStream::~HttpInputStream() {} -size_t HttpInputStream::read(void* buffer, size_t bytesToRead) { return pimpl_->read(buffer, bytesToRead); } //throw SysError, X; return "bytesToRead" bytes unless end of stream! +size_t HttpInputStream::tryRead(void* buffer, size_t bytesToRead) { return pimpl_->tryRead(buffer, bytesToRead); } size_t HttpInputStream::getBlockSize() const { return pimpl_->getBlockSize(); } -std::string HttpInputStream::readAll() { return bufferedLoad<std::string>(*pimpl_); } //throw SysError, X +std::string HttpInputStream::readAll(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw SysError, X +{ + return unbufferedLoad<std::string>([&](void* buffer, size_t bytesToRead) + { + const size_t bytesRead = pimpl_->tryRead(buffer, bytesToRead); //throw SysError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! + if (notifyUnbufferedIO) notifyUnbufferedIO(bytesRead); //throw X! + return bytesRead; + }, + pimpl_->getBlockSize()); //throw SysError, X +} namespace @@ -218,15 +241,16 @@ namespace std::unique_ptr<HttpInputStream::Impl> sendHttpRequestImpl(const Zstring& url, const std::string* postBuf /*issue POST if bound, GET otherwise*/, const std::string& contentType, //required for POST + const IoCallback& onPostBytesSent /*throw X*/, const Zstring& userAgent, - const Zstring& caCertFilePath /*optional: enable certificate validation*/, - const IoCallback& notifyUnbufferedIO) //throw SysError, X + const Zstring& caCertFilePath /*optional: enable certificate validation*/) //throw SysError, X { Zstring urlRed = url; //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop." for (int redirects = 0; redirects < 6; ++redirects) { - auto response = std::make_unique<HttpInputStream::Impl>(urlRed, postBuf, contentType, false /*disableGetCache*/, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError, X + auto response = std::make_unique<HttpInputStream::Impl>(urlRed, postBuf, contentType, onPostBytesSent, false /*disableGetCache*/, + userAgent, caCertFilePath); //throw SysError, X //https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection const int httpStatus = response->getStatusCode(); @@ -319,24 +343,30 @@ std::vector<std::pair<std::string, std::string>> zen::xWwwFormUrlDecode(const st } -HttpInputStream zen::sendHttpGet(const Zstring& url, const Zstring& userAgent, const Zstring& caCertFilePath, const IoCallback& notifyUnbufferedIO) //throw SysError, X +HttpInputStream zen::sendHttpGet(const Zstring& url, const Zstring& userAgent, const Zstring& caCertFilePath) //throw SysError { - return sendHttpRequestImpl(url, nullptr /*postBuf*/, "" /*contentType*/, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError, X, X + return sendHttpRequestImpl(url, nullptr /*postBuf*/, "" /*contentType*/, nullptr /*onPostBytesSent*/, userAgent, caCertFilePath); //throw SysError } HttpInputStream zen::sendHttpPost(const Zstring& url, const std::vector<std::pair<std::string, std::string>>& postParams, - const Zstring& userAgent, const Zstring& caCertFilePath, const IoCallback& notifyUnbufferedIO) //throw SysError, X + const IoCallback& notifyUnbufferedIO /*throw X*/, + const Zstring& userAgent, + const Zstring& caCertFilePath) //throw SysError, X { - return sendHttpPost(url, xWwwFormUrlEncode(postParams), "application/x-www-form-urlencoded", userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError, X + return sendHttpPost(url, xWwwFormUrlEncode(postParams), "application/x-www-form-urlencoded", notifyUnbufferedIO, userAgent, caCertFilePath); //throw SysError, X } -HttpInputStream zen::sendHttpPost(const Zstring& url, const std::string& postBuf, const std::string& contentType, - const Zstring& userAgent, const Zstring& caCertFilePath, const IoCallback& notifyUnbufferedIO) //throw SysError, X +HttpInputStream zen::sendHttpPost(const Zstring& url, + const std::string& postBuf, + const std::string& contentType, + const IoCallback& notifyUnbufferedIO /*throw X*/, + const Zstring& userAgent, + const Zstring& caCertFilePath) //throw SysError, X { - return sendHttpRequestImpl(url, &postBuf, contentType, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError, X + return sendHttpRequestImpl(url, &postBuf, contentType, notifyUnbufferedIO, userAgent, caCertFilePath); //throw SysError, X } @@ -347,10 +377,10 @@ bool zen::internetIsAlive() //noexcept auto response = std::make_unique<HttpInputStream::Impl>(Zstr("https://www.google.com/"), //https more appropriate than http for testing? (different ports!) nullptr /*postParams*/, "" /*contentType*/, + nullptr /*onPostBytesSent*/, true /*disableGetCache*/, Zstr("FreeFileSync"), - Zstring() /*caCertFilePath*/, - nullptr /*notifyUnbufferedIO*/); //throw SysError + Zstring() /*caCertFilePath*/); //throw SysError const int statusCode = response->getStatusCode(); //attention: google.com might redirect to https://consent.google.com => don't follow, just return "true"!!! @@ -12,17 +12,18 @@ namespace zen { -/* - thread-safe! (Window/Linux/macOS) - - Linux/macOS: init libcurl before use! */ +/* - Linux/macOS: init libcurl before use! + - safe to use on worker thread */ class HttpInputStream { public: - //support zen/serialize.h buffered input stream concept - size_t read(void* buffer, size_t bytesToRead); //throw SysError, X; return "bytesToRead" bytes unless end of stream! - std::string readAll(); //throw SysError, X + //zen/serialize.h unbuffered input stream concept: + size_t tryRead(void* buffer, size_t bytesToRead); //throw SysError; may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! size_t getBlockSize() const; + std::string readAll(const IoCallback& notifyUnbufferedIO /*throw X*/); //throw SysError, X + class Impl; HttpInputStream(std::unique_ptr<Impl>&& pimpl); HttpInputStream(HttpInputStream&&) noexcept = default; @@ -35,20 +36,17 @@ private: HttpInputStream sendHttpGet(const Zstring& url, const Zstring& userAgent, - const Zstring& caCertFilePath /*optional: enable certificate validation*/, - const IoCallback& notifyUnbufferedIO /*throw X*/); //throw SysError, X + const Zstring& caCertFilePath /*optional: enable certificate validation*/); //throw SysError HttpInputStream sendHttpPost(const Zstring& url, - const std::vector<std::pair<std::string, std::string>>& postParams, + const std::vector<std::pair<std::string, std::string>>& postParams, const IoCallback& notifyUnbufferedIO /*throw X*/, const Zstring& userAgent, - const Zstring& caCertFilePath /*optional: enable certificate validation*/, - const IoCallback& notifyUnbufferedIO /*throw X*/); //throw SysError, X + const Zstring& caCertFilePath /*optional: enable certificate validation*/); //throw SysError, X HttpInputStream sendHttpPost(const Zstring& url, - const std::string& postBuf, const std::string& contentType, + const std::string& postBuf, const std::string& contentType, const IoCallback& notifyUnbufferedIO /*throw X*/, const Zstring& userAgent, - const Zstring& caCertFilePath /*optional: enable certificate validation*/, - const IoCallback& notifyUnbufferedIO /*throw X*/); //throw SysError, X + const Zstring& caCertFilePath /*optional: enable certificate validation*/); //throw SysError, X bool internetIsAlive(); //noexcept std::wstring formatHttpError(int httpStatus); @@ -7,8 +7,6 @@ #ifndef I18_N_H_3843489325044253425456 #define I18_N_H_3843489325044253425456 -//#include <string> -//#include <cstdint> #include "globals.h" #include "string_tools.h" #include "format_unit.h" diff --git a/zen/open_ssl.cpp b/zen/open_ssl.cpp index 6dc13d3d..4494b76f 100644 --- a/zen/open_ssl.cpp +++ b/zen/open_ssl.cpp @@ -39,6 +39,8 @@ void zen::openSslInit() //excplicitly init OpenSSL on main thread: seems to initialize atomically! But it still might help to avoid issues: [[maybe_unused]] const int rv = ::OPENSSL_init_ssl(OPENSSL_INIT_SSL_DEFAULT | OPENSSL_INIT_NO_LOAD_CONFIG, nullptr); assert(rv == 1); //https://www.openssl.org/docs/man1.1.0/ssl/OPENSSL_init_ssl.html + + warn_static("probably should log") } @@ -240,7 +242,7 @@ std::string keyToStream(const EVP_PKEY* evp, RsaStreamType streamType, bool publ std::string keyStream(keyLen, '\0'); - if (::BIO_read(bio, &keyStream[0], keyLen) != keyLen) + if (::BIO_read(bio, keyStream.data(), keyLen) != keyLen) throw SysError(formatLastOpenSSLError("BIO_read")); return keyStream; } @@ -304,7 +306,7 @@ std::string keyToStream(const EVP_PKEY* evp, RsaStreamType streamType, bool publ std::string keyStream(keyLen, '\0'); - if (::BIO_read(bio, &keyStream[0], keyLen) != keyLen) + if (::BIO_read(bio, keyStream.data(), keyLen) != keyLen) throw SysError(formatLastOpenSSLError("BIO_read")); return keyStream; #endif @@ -354,9 +356,9 @@ std::string createSignature(const std::string& message, EVP_PKEY* privateKey) // std::string signature(sigLenMax, '\0'); size_t sigLen = sigLenMax; - if (::EVP_DigestSignFinal(mdctx, //EVP_MD_CTX* ctx - reinterpret_cast<unsigned char*>(&signature[0]), //unsigned char* sigret - &sigLen) != 1) //size_t* siglen + if (::EVP_DigestSignFinal(mdctx, //EVP_MD_CTX* ctx + reinterpret_cast<unsigned char*>(signature.data()), //unsigned char* sigret + &sigLen) != 1) //size_t* siglen throw SysError(formatLastOpenSSLError("EVP_DigestSignFinal")); signature.resize(sigLen); @@ -499,8 +501,8 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: const auto block2 = std::string("\0\0\0\1", 4) + passphrase; unsigned char key[2 * SHA_DIGEST_LENGTH] = {}; - ::SHA1(reinterpret_cast<const unsigned char*>(block1.c_str()), block1.size(), &key[0]); //no-fail - ::SHA1(reinterpret_cast<const unsigned char*>(block2.c_str()), block2.size(), &key[SHA_DIGEST_LENGTH]); // + ::SHA1(reinterpret_cast<const unsigned char*>(block1.c_str()), block1.size(), key); //no-fail + ::SHA1(reinterpret_cast<const unsigned char*>(block2.c_str()), block2.size(), key + SHA_DIGEST_LENGTH); // EVP_CIPHER_CTX* cipCtx = ::EVP_CIPHER_CTX_new(); if (!cipCtx) @@ -522,7 +524,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: int decLen1 = 0; if (::EVP_DecryptUpdate(cipCtx, //EVP_CIPHER_CTX* ctx - reinterpret_cast<unsigned char*>(&privateBlob[0]), //unsigned char* out + reinterpret_cast<unsigned char*>(privateBlob.data()), //unsigned char* out &decLen1, //int* outl reinterpret_cast<const unsigned char*>(privateBlobEnc.c_str()), //const unsigned char* in static_cast<int>(privateBlobEnc.size())) != 1) //int inl @@ -543,7 +545,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: macKeyBlob += passphrase; unsigned char macKey[SHA_DIGEST_LENGTH] = {}; - ::SHA1(reinterpret_cast<const unsigned char*>(macKeyBlob.c_str()), macKeyBlob.size(), &macKey[0]); //no-fail + ::SHA1(reinterpret_cast<const unsigned char*>(macKeyBlob.c_str()), macKeyBlob.size(), macKey); //no-fail auto numToBeString = [](size_t n) -> std::string { @@ -607,7 +609,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: { const std::string bytes = extractString(it, itEnd); - BIGNUM* bn = ::BN_bin2bn(reinterpret_cast<const unsigned char*>(&bytes[0]), static_cast<int>(bytes.size()), nullptr); + BIGNUM* bn = ::BN_bin2bn(reinterpret_cast<const unsigned char*>(bytes.c_str()), static_cast<int>(bytes.size()), nullptr); if (!bn) throw SysError(formatLastOpenSSLError("BN_bin2bn")); return std::unique_ptr<BIGNUM, BnFree>(bn); @@ -809,7 +811,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: if (::OSSL_PARAM_BLD_push_utf8_string(paramBld, OSSL_PKEY_PARAM_GROUP_NAME, groupName, 0) != 1) throw SysError(formatLastOpenSSLError("OSSL_PARAM_BLD_push_utf8_string(group)")); - if (::OSSL_PARAM_BLD_push_octet_string(paramBld, OSSL_PKEY_PARAM_PUB_KEY, &pointStream[0], pointStream.size()) != 1) + if (::OSSL_PARAM_BLD_push_octet_string(paramBld, OSSL_PKEY_PARAM_PUB_KEY, pointStream.data(), pointStream.size()) != 1) throw SysError(formatLastOpenSSLError("OSSL_PARAM_BLD_push_octet_string(pub)")); if (::OSSL_PARAM_BLD_push_BN(paramBld, OSSL_PKEY_PARAM_PRIV_KEY, pri.get()) != 1) @@ -861,7 +863,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: if (::EC_POINT_oct2point(ecGroup, //const EC_GROUP* group ecPoint, //EC_POINT* p - reinterpret_cast<const unsigned char*>(&pointStream[0]), //const unsigned char* buf + reinterpret_cast<const unsigned char*>(pointStream.c_str()), //const unsigned char* buf pointStream.size(), //size_t len nullptr) != 1) //BN_CTX* ctx throw SysError(formatLastOpenSSLError("EC_POINT_oct2point")); @@ -890,7 +892,7 @@ std::string zen::convertPuttyKeyToPkix(const std::string& keyStream, const std:: EVP_PKEY* evpPriv = ::EVP_PKEY_new_raw_private_key(EVP_PKEY_ED25519, //int type nullptr, //ENGINE* e - reinterpret_cast<const unsigned char*>(&priStream[0]), //const unsigned char* priv + reinterpret_cast<const unsigned char*>(priStream.c_str()), //const unsigned char* priv priStream.size()); //size_t len if (!evpPriv) throw SysError(formatLastOpenSSLError("EVP_PKEY_new_raw_private_key")); @@ -96,14 +96,12 @@ public: void showResult() { - const bool wasRunning = !watch_.isPaused(); - if (wasRunning) watch_.pause(); //don't include call to MessageBox()! - ZEN_ON_SCOPE_EXIT(if (wasRunning) watch_.resume()); - const int64_t timeMs = std::chrono::duration_cast<std::chrono::milliseconds>(watch_.elapsed()).count(); const std::string msg = numberTo<std::string>(timeMs) + " ms"; - std::clog << "Perf: duration: " << msg << '\n'; + std::clog << "Perf: duration: " << msg + '\n'; resultShown_ = true; + + watch_ = StopWatch(watch_.isPaused()); } private: diff --git a/zen/process_exec.cpp b/zen/process_exec.cpp index fb691151..a2c02eb0 100644 --- a/zen/process_exec.cpp +++ b/zen/process_exec.cpp @@ -122,7 +122,7 @@ std::pair<int /*exit code*/, std::string> processExecuteImpl(const Zstring& file argv.push_back(arg.c_str()); argv.push_back(nullptr); - /*int rv =*/::execv(argv[0], const_cast<char**>(&argv[0])); //only returns if an error occurred + /*int rv =*/::execv(argv[0], const_cast<char**>(argv.data())); //only returns if an error occurred //safe to cast away const: https://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html // "The statement about argv[] and envp[] being constants is included to make explicit to future // writers of language bindings that these objects are completely constant. Due to a limitation of @@ -206,8 +206,13 @@ std::pair<int /*exit code*/, std::string> processExecuteImpl(const Zstring& file THROW_LAST_SYS_ERROR("lseek"); guardTmpFile.dismiss(); - FileInput streamIn(fdTempFile, tempFilePath, nullptr /*notifyUnbufferedIO*/); //takes ownership! - std::string output = bufferedLoad<std::string>(streamIn); //throw FileError + FileInputPlain streamIn(fdTempFile, tempFilePath); //takes ownership! + + std::string output = unbufferedLoad<std::string>([&](void* buffer, size_t bytesToRead) + { + return streamIn.tryRead(buffer, bytesToRead); //throw FileError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! + }, + streamIn.getBlockSize()); //throw FileError if (!WIFEXITED(statusCode)) //signalled, crashed? throw SysError(formatSystemError("waitpid", WIFSIGNALED(statusCode) ? @@ -220,7 +225,7 @@ std::pair<int /*exit code*/, std::string> processExecuteImpl(const Zstring& file exitCode == 127) //details should have been streamed to STDERR: used by /bin/sh, e.g. failure to execute due to missing .so file throw SysError(utfTo<std::wstring>(trimCpy(output))); - return {exitCode, output}; + return {exitCode, std::move(output)}; } } diff --git a/zen/recycler.cpp b/zen/recycler.cpp index 4448fd60..26c27952 100644 --- a/zen/recycler.cpp +++ b/zen/recycler.cpp @@ -5,9 +5,7 @@ // ***************************************************************************** #include "recycler.h" -#include "file_access.h" - #include <sys/stat.h> #include <gio/gio.h> #include "scope_guard.h" @@ -17,7 +15,7 @@ using namespace zen; //*INDENT-OFF* -bool zen::recycleOrDeleteIfExists(const Zstring& itemPath) //throw FileError +void zen::moveToRecycleBin(const Zstring& itemPath) //throw FileError, RecycleBinUnavailable { GFile* file = ::g_file_new_for_path(itemPath.c_str()); //never fails according to docu ZEN_ON_SCOPE_EXIT(g_object_unref(file);) @@ -27,10 +25,6 @@ bool zen::recycleOrDeleteIfExists(const Zstring& itemPath) //throw FileError if (!::g_file_trash(file, nullptr, &error)) { - const std::optional<ItemType> type = itemStillExists(itemPath); //throw FileError - if (!type) - return false; - /* g_file_trash() can fail with different error codes/messages when trash is unavailable: Debian 8 (GLib 2.42): G_IO_ERROR_NOT_SUPPORTED: Unable to find or create trash directory CentOS 7 (GLib 2.56): G_IO_ERROR_FAILED: Unable to find or create trash directory for file.txt => localized! >:( @@ -42,7 +36,7 @@ bool zen::recycleOrDeleteIfExists(const Zstring& itemPath) //throw FileError //yes, the following is a cluster fuck, but what can you do? (error->code == G_IO_ERROR_FAILED && [&] { - for (const char* msgLoc : //translations from https://gitlab.gnome.org/GNOME/glib/-/tree/master/po + for (const char* msgLoc : //translations from https://gitlab.gnome.org/GNOME/glib/-/tree/main/po { "Unable to find or create trash directory for", "No s'ha pogut trobar o crear el directori de la paperera per", @@ -100,35 +94,12 @@ bool zen::recycleOrDeleteIfExists(const Zstring& itemPath) //throw FileError return false; }())); - if (trashUnavailable) //implement same behavior as on Windows: if recycler is not existing, delete permanently - { - if (*type == ItemType::folder) - removeDirectoryPlainRecursion(itemPath); //throw FileError - else - removeFilePlain(itemPath); //throw FileError - return true; - } + if (trashUnavailable) + throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(itemPath)), + formatGlibError("g_file_trash", error)); throw FileError(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(itemPath)), formatGlibError("g_file_trash", error)); } - return true; } //*INDENT-ON* - - -/* We really need access to a similar function to check whether a directory supports trashing and emit a warning if it does not! - - The following function looks perfect, alas it is restricted to local files and to the implementation of GIO only: - - gboolean _g_local_file_has_trash_dir(const char* dirpath, dev_t dir_dev); - See: http://www.netmite.com/android/mydroid/2.0/external/bluetooth/glib/gio/glocalfileinfo.h - - Just checking for "G_FILE_ATTRIBUTE_ACCESS_CAN_TRASH" is not correct, since we find in - http://www.netmite.com/android/mydroid/2.0/external/bluetooth/glib/gio/glocalfileinfo.c - - g_file_info_set_attribute_boolean (info, G_FILE_ATTRIBUTE_ACCESS_CAN_TRASH, - writable && parent_info->has_trash_dir); - - => We're NOT interested in whether the specified folder can be trashed, but whether it supports thrashing its child elements! (Only support, not actual write access!) - This renders G_FILE_ATTRIBUTE_ACCESS_CAN_TRASH useless for this purpose. */ diff --git a/zen/recycler.h b/zen/recycler.h index deb03a1c..79771321 100644 --- a/zen/recycler.h +++ b/zen/recycler.h @@ -18,23 +18,17 @@ namespace zen |Recycle Bin Access| -------------------- - Windows - ------- - -> Recycler API (IFileOperation) always available - -> COM needs to be initialized before calling any of these functions! CoInitializeEx/CoUninitialize + Windows: -> Recycler API (IFileOperation) always available + -> COM needs to be initialized before calling any of these functions! CoInitializeEx/CoUninitialize - Linux - ----- - Compiler flags: `pkg-config --cflags gio-2.0` - Linker flags: `pkg-config --libs gio-2.0` + Linux: Compiler flags: `pkg-config --cflags gio-2.0` + Linker flags: `pkg-config --libs gio-2.0` - Already included in package "gtk+-2.0"! */ - - -//move a file or folder to Recycle Bin (deletes permanently if recycler is not available) -> crappy semantics, but we have no choice thanks to Windows' design -bool recycleOrDeleteIfExists(const Zstring& itemPath); //throw FileError, return "true" if file/dir was actually deleted + Already included in package "gtk+-2.0"! */ +//fails if item is not existing (anymore) +void moveToRecycleBin(const Zstring& itemPath); //throw FileError, RecycleBinUnavailable } #endif //RECYCLER_H_18345067341545 diff --git a/zen/ring_buffer.h b/zen/ring_buffer.h index dfbb6493..f6cc3e5f 100644 --- a/zen/ring_buffer.h +++ b/zen/ring_buffer.h @@ -13,7 +13,7 @@ namespace zen { -//basically a std::deque<> but with a non-garbage implementation => circular buffer with std::vector<>-like exponential growth! +//like std::deque<> but with a non-garbage implementation: circular buffer with std::vector<>-like exponential growth! //https://stackoverflow.com/questions/39324192/why-is-an-stl-deque-not-implemented-as-just-a-circular-vector template <class T> @@ -34,8 +34,9 @@ public: using reference = T&; using const_reference = const T&; - size_t size() const { return size_; } - bool empty() const { return size_ == 0; } + size_t size () const { return size_; } + size_t capacity() const { return capacity_; } + bool empty () const { return size_ == 0; } reference front() { checkInvariants(); assert(!empty()); return getBufPtr()[bufStart_]; } const_reference front() const { checkInvariants(); assert(!empty()); return getBufPtr()[bufStart_]; } @@ -139,13 +140,13 @@ public: std::swap(size_, other.size_); } - void reserve(size_t minSize) //throw ? (strong exception-safety!) + void reserve(size_t minCapacity) //throw ? (strong exception-safety!) { checkInvariants(); - if (minSize > capacity_) + if (minCapacity > capacity_) { - const size_t newCapacity = std::max(minSize + minSize / 2, minSize); //no minimum capacity: just like std::vector<> implementation + const size_t newCapacity = std::max(minCapacity + minCapacity / 2, minCapacity); //no lower limit for capacity: just like std::vector<> RingBuffer newBuf(newCapacity); //throw ? @@ -184,13 +185,15 @@ public: Iterator& operator++() { ++offset_; return *this; } Iterator& operator--() { --offset_; return *this; } Iterator& operator+=(ptrdiff_t offset) { offset_ += offset; return *this; } - inline friend bool operator==(const Iterator& lhs, const Iterator& rhs) { assert(lhs.container_ == rhs.container_); return lhs.offset_ == rhs.offset_; } - inline friend ptrdiff_t operator-(const Iterator& lhs, const Iterator& rhs) { return lhs.offset_ - rhs.offset_; } - inline friend Iterator operator+(const Iterator& lhs, ptrdiff_t offset) { Iterator tmp(lhs); return tmp += offset; } Value& operator* () const { return (*container_)[offset_]; } Value* operator->() const { return &(*container_)[offset_]; } + inline friend Iterator operator+(const Iterator& lhs, ptrdiff_t offset) { Iterator tmp(lhs); return tmp += offset; } + inline friend ptrdiff_t operator-(const Iterator& lhs, const Iterator& rhs) { return lhs.offset_ - rhs.offset_; } + inline friend bool operator==(const Iterator& lhs, const Iterator& rhs) { assert(lhs.container_ == rhs.container_); return lhs.offset_ == rhs.offset_; } + inline friend std::strong_ordering operator<=>(const Iterator& lhs, const Iterator& rhs) { assert(lhs.container_ == rhs.container_); return lhs.offset_ <=> rhs.offset_; } + //GCC debug needs "operator<=" private: - Container* container_ = nullptr; + Container* container_ = nullptr; //iterator must be assignable ptrdiff_t offset_ = 0; }; @@ -214,8 +217,6 @@ private: rawMem_(static_cast<std::byte*>(::operator new (capacity * sizeof(T)))), //throw std::bad_alloc capacity_(capacity) {} - struct FreeStoreDelete { void operator()(std::byte* p) const { ::operator delete (p); } }; - /**/ T* getBufPtr() { return reinterpret_cast<T*>(rawMem_.get()); } const T* getBufPtr() const { return reinterpret_cast<T*>(rawMem_.get()); } @@ -227,20 +228,23 @@ private: static T* uninitializedMoveIfNoexcept(T* first, T* last, T* firstTrg, std::true_type ) { return std::uninitialized_move(first, last, firstTrg); } static T* uninitializedMoveIfNoexcept(T* first, T* last, T* firstTrg, std::false_type) { return std::uninitialized_copy(first, last, firstTrg); } //throw ? - void checkInvariants() const - { - assert(bufStart_ == 0 || bufStart_ < capacity_); - assert(size_ <= capacity_); - } - - size_t getBufPos(size_t offset) const //< capacity_ + size_t getBufPos(size_t offset) const { + //assert(offset < capacity_); -> redundant in this context size_t bufPos = bufStart_ + offset; if (bufPos >= capacity_) bufPos -= capacity_; return bufPos; } + void checkInvariants() const + { + assert(bufStart_ == 0 || bufStart_ < capacity_); + assert(size_ <= capacity_); + } + + struct FreeStoreDelete { void operator()(std::byte* p) const { ::operator delete (p); } }; + std::unique_ptr<std::byte, FreeStoreDelete> rawMem_; size_t capacity_ = 0; //as number of T size_t bufStart_ = 0; //< capacity_ diff --git a/zen/scope_guard.h b/zen/scope_guard.h index 1e4165be..4cc049a8 100644 --- a/zen/scope_guard.h +++ b/zen/scope_guard.h @@ -8,7 +8,6 @@ #define SCOPE_GUARD_H_8971632487321434 #include <cassert> -//#include <exception> #include "type_traits.h" #include "legacy_compiler.h" //std::uncaught_exceptions diff --git a/zen/serialize.h b/zen/serialize.h index 26202d96..53a6fc62 100644 --- a/zen/serialize.h +++ b/zen/serialize.h @@ -7,6 +7,7 @@ #ifndef SERIALIZE_H_839405783574356 #define SERIALIZE_H_839405783574356 +//#include <bit> #include <functional> #include "sys_error.h" //keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h! @@ -22,38 +23,44 @@ namespace zen binary container for data storage: must support "basic" std::vector interface (e.g. std::vector<std::byte>, std::string, Zbase<char>) --------------------------------- - | Buffered Input Stream Concept | + | Unbuffered Input Stream Concept | --------------------------------- - struct BufferedInputStream - { - size_t read(void* buffer, size_t bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream! + size_t getBlockSize(); //throw X + size_t tryRead(void* buffer, size_t bytesToRead); //throw X; may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + + ---------------------------------- + | Unbuffered Output Stream Concept | + ---------------------------------- + size_t getBlockSize(); //throw X + size_t tryWrite(const void* buffer, size_t bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0 + + =============================================================================================== - Optional: support stream-copying - -------------------------------- - size_t getBlockSize() const; - const IoCallback& notifyUnbufferedIO - }; + --------------------------------- + | Buffered Input Stream Concept | + --------------------------------- + size_t read(void* buffer, size_t bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream! ---------------------------------- | Buffered Output Stream Concept | ---------------------------------- - struct BufferedOutputStream - { - void write(const void* buffer, size_t bytesToWrite); //throw X + void write(const void* buffer, size_t bytesToWrite); //throw X */ - Optional: support stream-copying - -------------------------------- - const IoCallback& notifyUnbufferedIO - }; */ using IoCallback = std::function<void(int64_t bytesDelta)>; //throw X -//functions based on buffered stream abstraction -template <class BufferedInputStream, class BufferedOutputStream> -void bufferedStreamCopy(BufferedInputStream& streamIn, BufferedOutputStream& streamOut); //throw X +template <class BinContainer, class Function> +BinContainer unbufferedLoad(Function tryRead/*(void* buffer, size_t bytesToRead) throw X; may return short; only 0 means EOF*/, + size_t blockSize); //throw X + +template <class BinContainer, class Function> +void unbufferedSave(const BinContainer& cont, Function tryWrite /*(const void* buffer, size_t bytesToWrite) throw X; may return short*/, + size_t blockSize); //throw X + +template <class Function1, class Function2> +void unbufferedStreamCopy(Function1 tryRead /*(void* buffer, size_t bytesToRead) throw X; may return short; only 0 means EOF*/, size_t blockSizeIn, + Function2 tryWrite /*(const void* buffer, size_t bytesToWrite) throw X; may return short*/, size_t blockSizeOut); //throw X -template <class BinContainer, class BufferedInputStream> BinContainer -bufferedLoad(BufferedInputStream& streamIn); //throw X template <class N, class BufferedOutputStream> void writeNumber (BufferedOutputStream& stream, const N& num); // template <class C, class BufferedOutputStream> void writeContainer(BufferedOutputStream& stream, const C& str); //noexcept @@ -71,124 +78,277 @@ template < class BufferedInputStream> void readArray (BufferedInputSt struct IOCallbackDivider { - IOCallbackDivider(const IoCallback& notifyUnbufferedIO, int64_t& totalUnbufferedIO) : totalUnbufferedIO_(totalUnbufferedIO), notifyUnbufferedIO_(notifyUnbufferedIO) {} + IOCallbackDivider(const IoCallback& notifyUnbufferedIO, int64_t& totalBytesNotified) : + totalBytesNotified_(totalBytesNotified), + notifyUnbufferedIO_(notifyUnbufferedIO) { assert(totalBytesNotified == 0); } - void operator()(int64_t bytesDelta) + void operator()(int64_t bytesDelta) //throw X! { - if (notifyUnbufferedIO_) notifyUnbufferedIO_((totalUnbufferedIO_ - totalUnbufferedIO_ / 2 * 2 + bytesDelta) / 2); //throw X! - totalUnbufferedIO_ += bytesDelta; + if (notifyUnbufferedIO_) notifyUnbufferedIO_((totalBytesNotified_ + bytesDelta) / 2 - totalBytesNotified_ / 2); //throw X! + totalBytesNotified_ += bytesDelta; } private: - int64_t& totalUnbufferedIO_; + int64_t& totalBytesNotified_; const IoCallback& notifyUnbufferedIO_; }; +//------------------------------------------------------------------------------------- //buffered input/output stream reference implementations: -template <class BinContainer> struct MemoryStreamIn { - explicit MemoryStreamIn(const BinContainer& cont) : buffer_(cont) {} //this better be cheap! + explicit MemoryStreamIn(const std::string_view& stream) : memRef_(stream) {} + + MemoryStreamIn(std::string&&) = delete; //careful: do NOT store reference to a temporary! size_t read(void* buffer, size_t bytesToRead) //return "bytesToRead" bytes unless end of stream! { - using Byte = typename BinContainer::value_type; - static_assert(sizeof(Byte) == 1); - const size_t bytesRead = std::min(bytesToRead, buffer_.size() - pos_); - auto itFirst = buffer_.begin() + pos_; - std::copy(itFirst, itFirst + bytesRead, static_cast<Byte*>(buffer)); - pos_ += bytesRead; - return bytesRead; + const size_t junkSize = std::min(bytesToRead, memRef_.size() - pos_); + std::memcpy(buffer, memRef_.data() + pos_, junkSize); + pos_ += junkSize; + return junkSize; } size_t pos() const { return pos_; } private: - MemoryStreamIn (const MemoryStreamIn&) = delete; + //MemoryStreamIn (const MemoryStreamIn&) = delete; -> why not allow copying? MemoryStreamIn& operator=(const MemoryStreamIn&) = delete; - const BinContainer buffer_; + const std::string_view memRef_; size_t pos_ = 0; }; -template <class BinContainer> struct MemoryStreamOut { MemoryStreamOut() = default; void write(const void* buffer, size_t bytesToWrite) { - using Byte = typename BinContainer::value_type; - static_assert(sizeof(Byte) == 1); - buffer_.resize(buffer_.size() + bytesToWrite); - const auto it = static_cast<const Byte*>(buffer); - std::copy(it, it + bytesToWrite, buffer_.end() - bytesToWrite); + memBuf_.append(static_cast<const char*>(buffer), bytesToWrite); } - const BinContainer& ref() const { return buffer_; } - /**/ BinContainer& ref() { return buffer_; } + const std::string& ref() const { return memBuf_; } + /**/ std::string& ref() { return memBuf_; } private: MemoryStreamOut (const MemoryStreamOut&) = delete; MemoryStreamOut& operator=(const MemoryStreamOut&) = delete; - BinContainer buffer_; + std::string memBuf_; }; +//------------------------------------------------------------------------------------- +template <class Function> +struct BufferedInputStream +{ + BufferedInputStream(Function tryRead /*(void* buffer, size_t bytesToRead) throw X; may return short; only 0 means EOF*/, + size_t blockSize) : + tryRead_(tryRead), blockSize_(blockSize) {} + size_t read(void* buffer, size_t bytesToRead) //throw X; return "bytesToRead" bytes unless end of stream! + { + assert(memBuf_.size() >= blockSize_); + assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); + const auto bufStart = buffer; + for (;;) + { + const size_t junkSize = std::min(bytesToRead, bufPosEnd_ - bufPos_); + std::memcpy(buffer, memBuf_.data() + bufPos_ /*caveat: vector debug checks*/, junkSize); + bufPos_ += junkSize; + buffer = static_cast<std::byte*>(buffer) + junkSize; + bytesToRead -= junkSize; + + if (bytesToRead == 0) + break; + //-------------------------------------------------------------------- + const size_t bytesRead = tryRead_(memBuf_.data(), blockSize_); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + bufPos_ = 0; + bufPosEnd_ = bytesRead; + + if (bytesRead == 0) //end of file + break; + } + return static_cast<std::byte*>(buffer) - + static_cast<std::byte*>(bufStart); + } + +private: + BufferedInputStream (const BufferedInputStream&) = delete; + BufferedInputStream& operator=(const BufferedInputStream&) = delete; + Function tryRead_; + const size_t blockSize_; -//-----------------------implementation------------------------------- -template <class BufferedInputStream, class BufferedOutputStream> inline -void bufferedStreamCopy(BufferedInputStream& streamIn, //throw X - BufferedOutputStream& streamOut) // + size_t bufPos_ = 0; + size_t bufPosEnd_= 0; + std::vector<std::byte> memBuf_{blockSize_}; +}; + + +template <class Function> +struct BufferedOutputStream { - const size_t blockSize = streamIn.getBlockSize(); + BufferedOutputStream(Function tryWrite /*(const void* buffer, size_t bytesToWrite) throw X; may return short*/, + size_t blockSize) : + tryWrite_(tryWrite), blockSize_(blockSize) {} + + ~BufferedOutputStream() + { + } + + void write(const void* buffer, size_t bytesToWrite) //throw X + { + assert(memBuf_.size() >= blockSize_); + assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); + + for (;;) + { + const size_t junkSize = std::min(bytesToWrite, blockSize_ - (bufPosEnd_ - bufPos_)); + std::memcpy(memBuf_.data() + bufPosEnd_, buffer, junkSize); + bufPosEnd_ += junkSize; + buffer = static_cast<const std::byte*>(buffer) + junkSize; + bytesToWrite -= junkSize; + + if (bytesToWrite == 0) + return; + //-------------------------------------------------------------------- + bufPos_ += tryWrite_(memBuf_.data() + bufPos_, blockSize_); //throw X; may return short + + if (memBuf_.size() - bufPos_ < blockSize_ || //support memBuf_.size() > blockSize to avoid memmove()s + bufPos_ == bufPosEnd_) + { + std::memmove(memBuf_.data(), memBuf_.data() + bufPos_, bufPosEnd_ - bufPos_); + bufPosEnd_ -= bufPos_; + bufPos_ = 0; + } + } + } + + void flushBuffer() //throw X + { + assert(bufPosEnd_ - bufPos_ <= blockSize_); + assert(bufPos_ <= bufPosEnd_ && bufPosEnd_ <= memBuf_.size()); + while (bufPos_ != bufPosEnd_) + bufPos_ += tryWrite_(memBuf_.data() + bufPos_, bufPosEnd_ - bufPos_); //throw X + } + +private: + BufferedOutputStream (const BufferedOutputStream&) = delete; + BufferedOutputStream& operator=(const BufferedOutputStream&) = delete; + + Function tryWrite_; + const size_t blockSize_; + + size_t bufPos_ = 0; + size_t bufPosEnd_ = 0; + std::vector<std::byte> memBuf_{2 * /*=> mitigate memmove()*/ blockSize_}; //throw FileError +}; + +//------------------------------------------------------------------------------------- + +template <class BinContainer, class Function> inline +BinContainer unbufferedLoad(Function tryRead /*(void* buffer, size_t bytesToRead) throw X; may return short; only 0 means EOF*/, + size_t blockSize) //throw X +{ + static_assert(sizeof(typename BinContainer::value_type) == 1); //expect: bytes if (blockSize == 0) throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - std::vector<std::byte> buffer(blockSize); + BinContainer buf; for (;;) { - const size_t bytesRead = streamIn.read(&buffer[0], blockSize); //throw X; return "bytesToRead" bytes unless end of stream! - streamOut.write(&buffer[0], bytesRead); //throw X + warn_static("don't need zero-initialization!") + buf.resize(buf.size() + blockSize); + const size_t bytesRead = tryRead(buf.data() + buf.size() - blockSize, blockSize); //throw X; may return short; only 0 means EOF + buf.resize(buf.size() - blockSize + bytesRead); //caveat: unsigned arithmetics + + if (bytesRead == 0) //end of file + { + //caveat: memory consumption of returned string! + if (buf.capacity() > buf.size() * 3 / 2) //reference: in worst case, std::vector with growth factor 1.5 "wastes" 50% of its size as unused capacity + buf.shrink_to_fit(); //=> shrink if buffer is wasting more than that! - if (bytesRead < blockSize) //end of file - break; + return buf; + } } } -template <class BinContainer, class BufferedInputStream> inline -BinContainer bufferedLoad(BufferedInputStream& streamIn) //throw X +template <class BinContainer, class Function> inline +void unbufferedSave(const BinContainer& cont, + Function tryWrite /*(const void* buffer, size_t bytesToWrite) throw X; may return short*/, + size_t blockSize) //throw X { static_assert(sizeof(typename BinContainer::value_type) == 1); //expect: bytes - - const size_t blockSize = streamIn.getBlockSize(); if (blockSize == 0) throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); - BinContainer buffer; + const size_t bufPosEnd = cont.size(); + size_t bufPos = 0; + + while (bufPos < bufPosEnd) + bufPos += tryWrite(cont.data() + bufPos, std::min(bufPosEnd - bufPos, blockSize)); //throw X +} + + +template <class Function1, class Function2> inline +void unbufferedStreamCopy(Function1 tryRead /*(void* buffer, size_t bytesToRead) throw X; may return short; only 0 means EOF*/, + size_t blockSizeIn, + Function2 tryWrite /*(const void* buffer, size_t bytesToWrite) throw X; may return short*/, + size_t blockSizeOut) //throw X +{ + /* caveat: buffer block sizes might not be power of 2: + - f_iosize for network share on macOS + - libssh2 uses weird packet sizes like MAX_SFTP_OUTGOING_SIZE (30000), and will send incomplete packages if block size is not an exact multiple :( + => that's a problem because we want input/output sizes to be multiples of each other to help avoid the std::memmove() below */ +#if 0 + blockSizeIn = std::bit_ceil(blockSizeIn); + blockSizeOut = std::bit_ceil(blockSizeOut); +#endif + if (blockSizeIn <= 1 || blockSizeOut <= 1) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + + const size_t bufCapacity = blockSizeOut - 1 + blockSizeIn; + const size_t alignment = ::sysconf(_SC_PAGESIZE); //-1 on error => posix_memalign() will fail + assert(alignment >= sizeof(void*) && std::has_single_bit(alignment)); //required by posix_memalign() + std::byte* buf = nullptr; + errno = ::posix_memalign(reinterpret_cast<void**>(&buf), alignment, bufCapacity); + ZEN_ON_SCOPE_EXIT(::free(buf)); + + size_t bufPosEnd = 0; for (;;) { - buffer.resize(buffer.size() + blockSize); - const size_t bytesRead = streamIn.read(&*(buffer.end() - blockSize), blockSize); //throw X; return "blockSize" bytes unless end of stream! - if (bytesRead < blockSize) //end of file + const size_t bytesRead = tryRead(buf + bufPosEnd, blockSizeIn); //throw X; may return short; only 0 means EOF + + if (bytesRead == 0) //end of file { - buffer.resize(buffer.size() - (blockSize - bytesRead)); //caveat: unsigned arithmetics + size_t bufPos = 0; + while (bufPos < bufPosEnd) + bufPos += tryWrite(buf + bufPos, bufPosEnd - bufPos); //throw X; may return short + return; + } + else + { + bufPosEnd += bytesRead; - //caveat: memory consumption of returned string! - if (buffer.capacity() > buffer.size() * 3 / 2) //reference: in worst case, std::vector with growth factor 1.5 "wastes" 50% of its size as unused capacity - buffer.shrink_to_fit(); //=> shrink if buffer is wasting more than that! + size_t bufPos = 0; + while (bufPosEnd - bufPos >= blockSizeOut) + bufPos += tryWrite(buf + bufPos, blockSizeOut); //throw X; may return short - return buffer; + if (bufPos > 0) + { + bufPosEnd -= bufPos; + std::memmove(buf, buf + bufPos, bufPosEnd); + } } } } +//------------------------------------------------------------------------------------- template <class BufferedOutputStream> inline void writeArray(BufferedOutputStream& stream, const void* buffer, size_t len) @@ -232,7 +392,7 @@ template <class N, class BufferedInputStream> inline N readNumber(BufferedInputStream& stream) //throw SysErrorUnexpectedEos { static_assert(isArithmetic<N> || std::is_same_v<N, bool> || std::is_enum_v<N>); - N num{}; + N num; //uninitialized readArray(stream, &num, sizeof(N)); //throw SysErrorUnexpectedEos return num; } diff --git a/zen/socket.h b/zen/socket.h index d9517bd8..8e92b616 100644 --- a/zen/socket.h +++ b/zen/socket.h @@ -24,6 +24,7 @@ namespace zen using SocketType = int; const SocketType invalidSocket = -1; inline void closeSocket(SocketType s) { ::close(s); } +warn_static("log on error!") //Winsock needs to be initialized before calling any of these functions! (WSAStartup/WSACleanup) @@ -58,7 +59,7 @@ public: THROW_LAST_SYS_ERROR_WSA("socket"); ZEN_ON_SCOPE_FAIL(closeSocket(testSocket)); - + warn_static("support timeout! https://stackoverflow.com/questions/2597608/c-socket-connection-timeout") if (::connect(testSocket, ai.ai_addr, static_cast<int>(ai.ai_addrlen)) != 0) THROW_LAST_SYS_ERROR_WSA("connect"); diff --git a/zen/stream_buffer.h b/zen/stream_buffer.h index 8b8cd0d7..ee9e18fd 100644 --- a/zen/stream_buffer.h +++ b/zen/stream_buffer.h @@ -10,81 +10,78 @@ #include <condition_variable> #include "ring_buffer.h" #include "string_tools.h" +#include "thread.h" namespace zen { -/* implement streaming API on top of libcurl's icky callback-based design +/* implement streaming API on top of libcurl's icky callback-based design + + curl uses READBUFFER_SIZE download buffer size, but returns via a retarded sendf.c::chop_write() writing in small junks of CURL_MAX_WRITE_SIZE (16 kB) => support copying arbitrarily-large files: https://freefilesync.org/forum/viewtopic.php?t=4471 => maximum performance through async processing (prefetching + output buffer!) => cost per worker thread creation ~ 1/20 ms */ class AsyncStreamBuffer { public: - explicit AsyncStreamBuffer(size_t bufferSize) : bufSize_(bufferSize) { ringBuf_.reserve(bufferSize); } + explicit AsyncStreamBuffer(size_t capacity) { ringBuf_.reserve(capacity); } //context of input thread, blocking - //return "bytesToRead" bytes unless end of stream! - size_t read(void* buffer, size_t bytesToRead) //throw <write error> + size_t read(void* buffer, size_t bytesToRead) //throw <write error>; return "bytesToRead" bytes unless end of stream! { - if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! - throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + std::unique_lock dummy(lockStream_); + const auto bufStart = buffer; - auto it = static_cast<std::byte*>(buffer); - const auto itEnd = it + bytesToRead; - - for (std::unique_lock dummy(lockStream_); it != itEnd;) + while (bytesToRead > 0) { - assert(!errorRead_); - conditionBytesWritten_.wait(dummy, [this] { return errorWrite_ || !ringBuf_.empty() || eof_; }); - - if (errorWrite_) - std::rethrow_exception(errorWrite_); //throw <write error> - - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), ringBuf_.size()); - ringBuf_.extract_front(it, it + junkSize); - it += junkSize; - - conditionBytesRead_.notify_all(); - - if (eof_) //end of file + const size_t bytesRead = tryReadImpl(dummy, buffer, bytesToRead); //throw <write error> + if (bytesRead == 0) //end of file break; + conditionBytesRead_.notify_all(); + buffer = static_cast<std::byte*>(buffer) + bytesRead; + bytesToRead -= bytesRead; } + return static_cast<std::byte*>(buffer) - + static_cast<std::byte*>(bufStart); + } - const size_t bytesRead = it - static_cast<std::byte*>(buffer); - totalBytesRead_ += bytesRead; + //context of input thread, blocking + size_t tryRead(void* buffer, size_t bytesToRead) //throw <write error>; may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + { + size_t bytesRead = 0; + { + std::unique_lock dummy(lockStream_); + bytesRead = tryReadImpl(dummy, buffer, bytesToRead); + } + if (bytesRead > 0) + conditionBytesRead_.notify_all(); //...*outside* the lock return bytesRead; } //context of output thread, blocking void write(const void* buffer, size_t bytesToWrite) //throw <read error> { - totalBytesWritten_ += bytesToWrite; //bytes already processed as far as raw FTP access is concerned - - auto it = static_cast<const std::byte*>(buffer); - const auto itEnd = it + bytesToWrite; - - for (std::unique_lock dummy(lockStream_); it != itEnd;) + std::unique_lock dummy(lockStream_); + while (bytesToWrite > 0) { - assert(!eof_ && !errorWrite_); - /* => can't use InterruptibleThread's interruptibleWait() :( - -> AsyncStreamBuffer is used for input and output streaming - => both AsyncStreamBuffer::write()/read() would have to implement interruptibleWait() - => one of these usually called from main thread - => but interruptibleWait() cannot be called from main thread! */ - conditionBytesRead_.wait(dummy, [this] { return errorRead_ || ringBuf_.size() < bufSize_; }); - - if (errorRead_) - std::rethrow_exception(errorRead_); //throw <read error> - - const size_t junkSize = std::min(static_cast<size_t>(itEnd - it), bufSize_ - ringBuf_.size()); - ringBuf_.insert_back(it, it + junkSize); - it += junkSize; - + const size_t bytesWritten = tryWriteWhileImpl(dummy, buffer, bytesToWrite); //throw <read error> conditionBytesWritten_.notify_all(); + buffer = static_cast<const std::byte*>(buffer) + bytesWritten; + bytesToWrite -= bytesWritten; } } + //context of output thread, blocking + size_t tryWrite(const void* buffer, size_t bytesToWrite) //throw <read error>; may return short! CONTRACT: bytesToWrite > 0 + { + size_t bytesWritten = 0; + { + std::unique_lock dummy(lockStream_); + bytesWritten = tryWriteWhileImpl(dummy, buffer, bytesToWrite); + } + conditionBytesWritten_.notify_all(); //...*outside* the lock + return bytesWritten; + } + //context of output thread void closeStream() { @@ -101,7 +98,7 @@ public: { { std::lock_guard dummy(lockStream_); - assert(!errorRead_); + assert(error && !errorRead_); if (!errorRead_) errorRead_ = error; } @@ -113,13 +110,16 @@ public: { { std::lock_guard dummy(lockStream_); - assert(!errorWrite_); + assert(error && !errorWrite_); if (!errorWrite_) errorWrite_ = error; } conditionBytesWritten_.notify_all(); } +#if 0 + //function not needed: when writing is completed successfully, no further error can occur! + // => caveat: writing is NOT done (yet) when closeStream() is called! //context of *output* thread void checkReadErrors() //throw <read error> { @@ -128,7 +128,8 @@ public: std::rethrow_exception(errorRead_); //throw <read error> } -#if 0 //function not needed: when EOF is reached (without errors), reading is done => no further error can occur! + //function not needed: when EOF is reached (without errors), reading is done => no further error can occur! + //context of *input* thread void checkWriteErrors() //throw <write error> { std::lock_guard dummy(lockStream_); @@ -144,7 +145,53 @@ private: AsyncStreamBuffer (const AsyncStreamBuffer&) = delete; AsyncStreamBuffer& operator=(const AsyncStreamBuffer&) = delete; - const size_t bufSize_; + //context of input thread, blocking + size_t tryReadImpl(std::unique_lock<std::mutex>& ul, void* buffer, size_t bytesToRead) //throw <write error>; may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! + { + if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + + assert(isLocked(lockStream_)); + assert(!errorRead_); + + conditionBytesWritten_.wait(ul, [this] { return errorWrite_ || !ringBuf_.empty() || eof_; }); + + if (errorWrite_) + std::rethrow_exception(errorWrite_); //throw <write error> + + const size_t junkSize = std::min(bytesToRead, ringBuf_.size()); + ringBuf_.extract_front(static_cast<std::byte*>(buffer), + static_cast<std::byte*>(buffer)+ junkSize); + totalBytesRead_ += junkSize; + return junkSize; + } + + //context of output thread, blocking + size_t tryWriteWhileImpl(std::unique_lock<std::mutex>& ul, const void* buffer, size_t bytesToWrite) //throw <read error>; may return short! CONTRACT: bytesToWrite > 0 + { + if (bytesToWrite == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ':' + numberTo<std::string>(__LINE__)); + + assert(isLocked(lockStream_)); + assert(!eof_ && !errorWrite_); + /* => can't use InterruptibleThread's interruptibleWait() :( + -> AsyncStreamBuffer is used for input and output streaming + => both AsyncStreamBuffer::write()/read() would have to implement interruptibleWait() + => one of these usually called from main thread + => but interruptibleWait() cannot be called from main thread! */ + conditionBytesRead_.wait(ul, [this] { return errorRead_ || ringBuf_.size() < ringBuf_.capacity(); }); + + if (errorRead_) + std::rethrow_exception(errorRead_); //throw <read error> + + const size_t junkSize = std::min(bytesToWrite, ringBuf_.capacity() - ringBuf_.size()); + + ringBuf_.insert_back(static_cast<const std::byte*>(buffer), + static_cast<const std::byte*>(buffer) + junkSize); + totalBytesWritten_ += junkSize; + return junkSize; + } + std::mutex lockStream_; RingBuffer<std::byte> ringBuf_; //prefetch/output buffer bool eof_ = false; diff --git a/zen/string_base.h b/zen/string_base.h index e18a0f16..a87827e6 100644 --- a/zen/string_base.h +++ b/zen/string_base.h @@ -61,11 +61,11 @@ protected: ~StorageDeepCopy() {} Char* create(size_t size) { return create(size, size); } - Char* create(size_t size, size_t minCapacity) + Char* create(size_t size, size_t capacity) { - assert(size <= minCapacity); - const size_t newCapacity = AP::calcCapacity(minCapacity); - assert(newCapacity >= minCapacity); + assert(size <= capacity); + const size_t newCapacity = AP::calcCapacity(capacity); + assert(newCapacity >= capacity); Descriptor* const newDescr = static_cast<Descriptor*>(this->allocate(sizeof(Descriptor) + (newCapacity + 1) * sizeof(Char))); //throw std::bad_alloc new (newDescr) Descriptor(size, newCapacity); @@ -124,18 +124,18 @@ protected: ~StorageRefCountThreadSafe() {} Char* create(size_t size) { return create(size, size); } - Char* create(size_t size, size_t minCapacity) + Char* create(size_t size, size_t capacity) { - assert(size <= minCapacity); + assert(size <= capacity); - if (minCapacity == 0) //perf: avoid memory allocation for empty string + if (capacity == 0) //perf: avoid memory allocation for empty string { ++globalEmptyString.descr.refCount; return &globalEmptyString.nullTerm; } - const size_t newCapacity = AP::calcCapacity(minCapacity); - assert(newCapacity >= minCapacity); + const size_t newCapacity = AP::calcCapacity(capacity); + assert(newCapacity >= capacity); Descriptor* const newDescr = static_cast<Descriptor*>(this->allocate(sizeof(Descriptor) + (newCapacity + 1) * sizeof(Char))); //throw std::bad_alloc new (newDescr) Descriptor(size, newCapacity); @@ -259,6 +259,7 @@ public: size_t length() const; size_t size () const { return length(); } const Char* c_str() const { return rawStr_; } //C-string format with 0-termination + /**/ Char* data() { return &*begin(); } const Char& operator[](size_t pos) const; /**/ Char& operator[](size_t pos); bool empty() const { return length() == 0; } @@ -558,7 +559,7 @@ void Zbase<Char, SP>::reserve(size_t minCapacity) //make unshared and check capa //allocate a new string const size_t len = length(); Char* newStr = this->create(len, std::max(len, minCapacity)); //reserve() must NEVER shrink the string: logical const! - std::copy(rawStr_, rawStr_ + len + 1, newStr); //include 0-termination + std::copy(rawStr_, rawStr_ + len + 1 /*0-termination*/, newStr); this->destroy(rawStr_); rawStr_ = newStr; diff --git a/zen/string_tools.h b/zen/string_tools.h index 181a3951..364a9a26 100644 --- a/zen/string_tools.h +++ b/zen/string_tools.h @@ -263,7 +263,7 @@ bool equalString(const S& lhs, const T& rhs) template <class S, class T> inline bool equalAsciiNoCase(const S& lhs, const T& rhs) { - //assert(isAsciiString(lhs) || isAsciiString(rhs)); + //assert(isAsciiString(lhs) || isAsciiString(rhs)); -> no, too strict (e.g. comparing file extensions ASCII-CI) const size_t lhsLen = strLength(lhs); return lhsLen == strLength(rhs) && impl::strcmpAsciiNoCase(strBegin(lhs), strBegin(rhs), lhsLen) == std::weak_ordering::equivalent; } @@ -627,10 +627,14 @@ S printNumber(const T& format, const Num& number) //format a single number using #endif static_assert(std::is_same_v<GetCharTypeT<S>, GetCharTypeT<T>>); - GetCharTypeT<S> buf[128]; //zero-initialize? - const int charsWritten = impl::saferPrintf(buf, std::size(buf), strBegin(format), number); + S buf(128, static_cast<GetCharTypeT<S>>('0')); + const int charsWritten = impl::saferPrintf(buf.data(), buf.size(), strBegin(format), number); - return 0 < charsWritten && charsWritten < std::ssize(buf) ? S(buf, charsWritten) : S(); + if (makeUnsigned(charsWritten) > buf.size()) + return S(); + + buf.resize(charsWritten); + return buf; } diff --git a/zen/string_traits.h b/zen/string_traits.h index 31c8c12c..240dbeac 100644 --- a/zen/string_traits.h +++ b/zen/string_traits.h @@ -36,8 +36,8 @@ namespace zen //reference a sub-string for consumption by zen string_tools //=> std::string_view seems decent, but of course fucks up in one regard: construction -template <class Iterator> auto makeStringView(Iterator first, Iterator last); //e.g. this constructor is not available (at least on clang) -template <class Iterator> auto makeStringView(Iterator first, size_t len); +template <class Iterator> auto makeStringView(Iterator first, Iterator last); //this constructor is not available (at least on clang) +template <class Iterator> auto makeStringView(Iterator first, size_t len); //std::string_view(char*, int) fails to compile! expected size_t as second parameter diff --git a/zen/symlink_target.h b/zen/symlink_target.h index ada4e358..44c15ab2 100644 --- a/zen/symlink_target.h +++ b/zen/symlink_target.h @@ -44,13 +44,13 @@ zen::SymlinkRawContent getSymlinkRawContent_impl(const Zstring& linkPath) //thro const size_t bufSize = 10000; std::vector<char> buf(bufSize); - const ssize_t bytesWritten = ::readlink(linkPath.c_str(), &buf[0], bufSize); + const ssize_t bytesWritten = ::readlink(linkPath.c_str(), buf.data(), bufSize); if (bytesWritten < 0) THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(linkPath)), "readlink"); if (bytesWritten >= static_cast<ssize_t>(bufSize)) //detect truncation; not an error for readlink! throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(linkPath)), formatSystemError("readlink", L"", L"Buffer truncated.")); - return {Zstring(&buf[0], bytesWritten)}; //readlink does not append 0-termination! + return {.targetPath = Zstring(buf.data(), bytesWritten)}; //readlink does not append 0-termination! } diff --git a/zen/sys_error.h b/zen/sys_error.h index 6d03f299..73a92343 100644 --- a/zen/sys_error.h +++ b/zen/sys_error.h @@ -42,6 +42,7 @@ private: +//better leave it as a macro (see comment in file_error.h) #define THROW_LAST_SYS_ERROR(functionName) \ do { const ErrorCode ecInternal = getLastError(); throw zen::SysError(formatSystemError(functionName, ecInternal)); } while (false) diff --git a/zen/sys_info.cpp b/zen/sys_info.cpp index c57464bc..87e07f91 100644 --- a/zen/sys_info.cpp +++ b/zen/sys_info.cpp @@ -43,7 +43,7 @@ Zstring zen::getLoginUser() //throw FileError passwd* pwEntry = nullptr; if (const int rv = ::getpwuid_r(userIdNo, //uid_t uid &buf2, //struct passwd* pwd - &buf[0], //char* buf + buf.data(), //char* buf buf.size(), //size_t buflen &pwEntry); //struct passwd** result rv != 0 || !pwEntry) @@ -82,10 +82,10 @@ Zstring zen::getUserDescription() //throw FileError const Zstring computerName = []() -> Zstring //throw FileError { std::vector<char> buf(10000); - if (::gethostname(&buf[0], buf.size()) != 0) + if (::gethostname(buf.data(), buf.size()) != 0) THROW_LAST_FILE_ERROR(_("Cannot get process information."), "gethostname"); - Zstring hostName = &buf[0]; + Zstring hostName = buf.data(); if (endsWithAsciiNoCase(hostName, ".local")) //strip fluff (macOS) => apparently not added on Linux? hostName = beforeLast(hostName, '.', IfNotFoundReturn::none); @@ -204,7 +204,7 @@ Zstring zen::getUserHome() //throw FileError passwd* pwEntry = nullptr; if (const int rv = ::getpwnam_r(loginUser.c_str(), //const char *name &buf2, //struct passwd* pwd - &buf[0], //char* buf + buf.data(), //char* buf buf.size(), //size_t buflen &pwEntry); //struct passwd** result rv != 0 || !pwEntry) diff --git a/zen/sys_version.cpp b/zen/sys_version.cpp index 7355ef1a..d187e0f0 100644 --- a/zen/sys_version.cpp +++ b/zen/sys_version.cpp @@ -88,7 +88,7 @@ OsVersion zen::getOsVersion() } catch (const SysError& e) { - std::cerr << utfTo<std::string>(e.toString()) << '\n'; + std::cerr << utfTo<std::string>(e.toString()) + '\n'; return OsVersionDetail{}; //sigh, it's a jungle out there: https://freefilesync.org/forum/viewtopic.php?t=7276 } }(); diff --git a/zen/thread.h b/zen/thread.h index abdc6da0..931f2c0d 100644 --- a/zen/thread.h +++ b/zen/thread.h @@ -72,6 +72,7 @@ void interruptibleSleep(const std::chrono::duration<Rep, Period>& relTime); //th void setCurrentThreadName(const Zstring& threadName); bool runningOnMainThread(); + //------------------------------------------------------------------------------------------ /* std::async replacement without crappy semantics: @@ -272,14 +272,14 @@ Zstring formatTime(const Zchar* format, const TimeComp& tc) std::mktime(&ctc); //unfortunately std::strftime() needs all elements of "struct tm" filled, e.g. tm_wday, tm_yday //note: although std::mktime() explicitly expects "local time", calculating weekday and day of year *should* be time-zone and DST independent - Zstring buffer(256, Zstr('\0')); + Zstring buf(256, Zstr('\0')); //strftime() craziness on invalid input: // VS 2010: CRASH unless "_invalid_parameter_handler" is set: https://docs.microsoft.com/en-us/cpp/c-runtime-library/parameter-validation // GCC: returns 0, apparently no crash. Still, considering some clib maintainer's comments, we should expect the worst! // Windows: avoid char-based strftime() which uses ANSI encoding! (e.g. Greek letters for AM/PM) - const size_t charsWritten = std::strftime(&buffer[0], buffer.size(), format, &ctc); - buffer.resize(charsWritten); - return buffer; + const size_t charsWritten = std::strftime(buf.data(), buf.size(), format, &ctc); + buf.resize(charsWritten); + return buf; } diff --git a/zen/zlib_wrap.cpp b/zen/zlib_wrap.cpp index e87a284f..28b85c5c 100644 --- a/zen/zlib_wrap.cpp +++ b/zen/zlib_wrap.cpp @@ -8,8 +8,9 @@ //Windows: use the SAME zlib version that wxWidgets is linking against! //C:\Data\Projects\wxWidgets\Source\src\zlib\zlib.h //Linux/macOS: use zlib system header for both wxWidgets and libcurl (zlib is required for HTTP, SFTP) // => don't compile wxWidgets with: --with-zlib=builtin -#include <zlib.h> //https://www.zlib.net/manual.html -#include <zen/scope_guard.h> +#include <zlib.h> +#include "scope_guard.h" +#include "serialize.h" using namespace zen; @@ -20,9 +21,9 @@ std::wstring getZlibErrorLiteral(int sc) { switch (sc) { - ZEN_CHECK_CASE_FOR_CONSTANT(Z_OK); - ZEN_CHECK_CASE_FOR_CONSTANT(Z_STREAM_END); ZEN_CHECK_CASE_FOR_CONSTANT(Z_NEED_DICT); + ZEN_CHECK_CASE_FOR_CONSTANT(Z_STREAM_END); + ZEN_CHECK_CASE_FOR_CONSTANT(Z_OK); ZEN_CHECK_CASE_FOR_CONSTANT(Z_ERRNO); ZEN_CHECK_CASE_FOR_CONSTANT(Z_STREAM_ERROR); ZEN_CHECK_CASE_FOR_CONSTANT(Z_DATA_ERROR); @@ -34,16 +35,15 @@ std::wstring getZlibErrorLiteral(int sc) return replaceCpy<std::wstring>(L"zlib error %x", L"%x", numberTo<std::wstring>(sc)); } } -} -size_t zen::impl::zlib_compressBound(size_t len) +size_t zlib_compressBound(size_t len) { return ::compressBound(static_cast<uLong>(len)); //upper limit for buffer size, larger than input size!!! } -size_t zen::impl::zlib_compress(const void* src, size_t srcLen, void* trg, size_t trgLen, int level) //throw SysError +size_t zlib_compress(const void* src, size_t srcLen, void* trg, size_t trgLen, int level) //throw SysError { uLongf bufSize = static_cast<uLong>(trgLen); const int rv = ::compress2(static_cast<Bytef*>(trg), //Bytef* dest @@ -61,7 +61,7 @@ size_t zen::impl::zlib_compress(const void* src, size_t srcLen, void* trg, size_ } -size_t zen::impl::zlib_decompress(const void* src, size_t srcLen, void* trg, size_t trgLen) //throw SysError +size_t zlib_decompress(const void* src, size_t srcLen, void* trg, size_t trgLen) //throw SysError { uLongf bufSize = static_cast<uLong>(trgLen); const int rv = ::uncompress(static_cast<Bytef*>(trg), //Bytef* dest @@ -77,13 +77,80 @@ size_t zen::impl::zlib_decompress(const void* src, size_t srcLen, void* trg, siz return bufSize; } +} + + +#undef compress //mitigate zlib macro shit... + +std::string zen::compress(const std::string_view& stream, int level) //throw SysError +{ + std::string output; + if (!stream.empty()) //don't dereference iterator into empty container! + { + //save uncompressed stream size for decompression + const uint64_t uncompressedSize = stream.size(); //use portable number type! + output.resize(sizeof(uncompressedSize)); + std::memcpy(output.data(), &uncompressedSize, sizeof(uncompressedSize)); + + const size_t bufferEstimate = zlib_compressBound(stream.size()); //upper limit for buffer size, larger than input size!!! + + output.resize(output.size() + bufferEstimate); + + const size_t bytesWritten = zlib_compress(stream.data(), + stream.size(), + output.data() + output.size() - bufferEstimate, + bufferEstimate, + level); //throw SysError + if (bytesWritten < bufferEstimate) + output.resize(output.size() - bufferEstimate + bytesWritten); //caveat: unsigned arithmetics + //caveat: physical memory consumption still *unchanged*! + } + return output; +} + + +std::string zen::decompress(const std::string_view& stream) //throw SysError +{ + std::string output; + if (!stream.empty()) //don't dereference iterator into empty container! + { + //retrieve size of uncompressed data + uint64_t uncompressedSize = 0; //use portable number type! + if (stream.size() < sizeof(uncompressedSize)) + throw SysError(L"zlib error: stream size < 8"); + + std::memcpy(&uncompressedSize, stream.data(), sizeof(uncompressedSize)); + + //attention: output MUST NOT be empty! Else it will pass a nullptr to zlib_decompress() => Z_STREAM_ERROR although "uncompressedSize == 0"!!! + if (uncompressedSize == 0) //cannot be 0: compress() directly maps empty -> empty container skipping zlib! + throw SysError(L"zlib error: uncompressed size == 0"); + + try + { + output.resize(static_cast<size_t>(uncompressedSize)); //throw std::bad_alloc + } + //most likely this is due to data corruption: + catch (const std::length_error& e) { throw SysError(L"zlib error: " + _("Out of memory.") + L' ' + utfTo<std::wstring>(e.what())); } + catch (const std::bad_alloc& e) { throw SysError(L"zlib error: " + _("Out of memory.") + L' ' + utfTo<std::wstring>(e.what())); } + + const size_t bytesWritten = zlib_decompress(stream.data() + sizeof(uncompressedSize), + stream.size() - sizeof(uncompressedSize), + output.data(), + static_cast<size_t>(uncompressedSize)); //throw SysError + if (bytesWritten != static_cast<size_t>(uncompressedSize)) + throw SysError(formatSystemError("zlib_decompress", L"", L"bytes written != uncompressed size.")); + } + return output; +} class InputStreamAsGzip::Impl { public: - Impl(const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/) : //throw SysError; returning 0 signals EOF: Posix read() semantics - readBlock_(readBlock) + Impl(const std::function<size_t(void* buffer, size_t bytesToRead)>& tryReadBlock /*throw X; may return short, only 0 means EOF!*/, + size_t blockSize) : //throw SysError + tryReadBlock_(tryReadBlock), + blockSize_(blockSize) { const int windowBits = MAX_WBITS + 16; //"add 16 to windowBits to write a simple gzip header" @@ -105,6 +172,7 @@ public: { [[maybe_unused]] const int rv = ::deflateEnd(&gzipStream_); assert(rv == Z_OK); + warn_static("log on error") } size_t read(void* buffer, size_t bytesToRead) //throw SysError, X; return "bytesToRead" bytes unless end of stream! @@ -117,20 +185,18 @@ public: for (;;) { + //refill input buffer once avail_in == 0: https://www.zlib.net/manual.html if (gzipStream_.avail_in == 0 && !eof_) { - if (bufIn_.size() < bytesToRead) - bufIn_.resize(bytesToRead); - - const size_t bytesRead = readBlock_(&bufIn_[0], bufIn_.size()); //throw X; returning 0 signals EOF: Posix read() semantics - gzipStream_.next_in = reinterpret_cast<z_const Bytef*>(&bufIn_[0]); + const size_t bytesRead = tryReadBlock_(bufIn_.data(), blockSize_); //throw X; may return short, only 0 means EOF! + gzipStream_.next_in = reinterpret_cast<z_const Bytef*>(bufIn_.data()); gzipStream_.avail_in = static_cast<uInt>(bytesRead); if (bytesRead == 0) eof_ = true; } const int rv = ::deflate(&gzipStream_, eof_ ? Z_FINISH : Z_NO_FLUSH); - if (rv == Z_STREAM_END) + if (eof_ && rv == Z_STREAM_END) return bytesToRead - gzipStream_.avail_out; if (rv != Z_OK) throw SysError(formatSystemError("zlib deflate", getZlibErrorLiteral(rv), L"")); @@ -140,34 +206,41 @@ public: } } + size_t getBlockSize() const { return blockSize_; } //returning input blockSize_ makes sense for low compression ratio + private: - const std::function<size_t(void* buffer, size_t bytesToRead)> readBlock_; //throw X + const std::function<size_t(void* buffer, size_t bytesToRead)> tryReadBlock_; //throw X + const size_t blockSize_; bool eof_ = false; - std::vector<std::byte> bufIn_; + std::vector<std::byte> bufIn_{blockSize_}; z_stream gzipStream_ = {}; }; -zen::InputStreamAsGzip::InputStreamAsGzip(const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/) : pimpl_(std::make_unique<Impl>(readBlock)) {} //throw SysError -zen::InputStreamAsGzip::~InputStreamAsGzip() {} -size_t zen::InputStreamAsGzip::read(void* buffer, size_t bytesToRead) { return pimpl_->read(buffer, bytesToRead); } //throw SysError, X +InputStreamAsGzip::InputStreamAsGzip(const std::function<size_t(void* buffer, size_t bytesToRead)>& tryReadBlock /*throw X*/, size_t blockSize) : + pimpl_(std::make_unique<Impl>(tryReadBlock, blockSize)) {} //throw SysError + +InputStreamAsGzip::~InputStreamAsGzip() {} + +size_t InputStreamAsGzip::getBlockSize() const { return pimpl_->getBlockSize(); } +size_t InputStreamAsGzip::read(void* buffer, size_t bytesToRead) { return pimpl_->read(buffer, bytesToRead); } //throw SysError, X -std::string zen::compressAsGzip(const void* buffer, size_t bufSize) //throw SysError + +std::string zen::compressAsGzip(const std::string_view& stream) //throw SysError { - struct MemoryStreamAsGzip : InputStreamAsGzip + MemoryStreamIn memStream(stream); + + auto tryReadBlock = [&](void* buffer, size_t bytesToRead) //may return short, only 0 means EOF! { - explicit MemoryStreamAsGzip(const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X*/) : InputStreamAsGzip(readBlock) {} //throw SysError - static size_t getBlockSize() { return 128 * 1024; } //InputStreamAsGzip has no idea what it's wrapping => has no getBlockSize() member! + return memStream.read(buffer, bytesToRead); //return "bytesToRead" bytes unless end of stream! }; - MemoryStreamAsGzip gzipStream([&](void* bufIn, size_t bytesToRead) //throw SysError + InputStreamAsGzip gzipStream(tryReadBlock, 1024 * 1024 /*blockSize*/); //throw SysError + + return unbufferedLoad<std::string>([&](void* buffer, size_t bytesToRead) { - const size_t bytesRead = std::min(bufSize, bytesToRead); - std::memcpy(bufIn, buffer, bytesRead); - buffer = static_cast<const char*>(buffer) + bytesRead; - bufSize -= bytesRead; - return bytesRead; //returning 0 signals EOF: Posix read() semantics - }); - return bufferedLoad<std::string>(gzipStream); //throw SysError + return gzipStream.read(buffer, bytesToRead); //throw SysError; return "bytesToRead" bytes unless end of stream! + }, + gzipStream.getBlockSize()); //throw SysError } diff --git a/zen/zlib_wrap.h b/zen/zlib_wrap.h index 41d7428a..d672707b 100644 --- a/zen/zlib_wrap.h +++ b/zen/zlib_wrap.h @@ -7,7 +7,7 @@ #ifndef ZLIB_WRAP_H_428597064566 #define ZLIB_WRAP_H_428597064566 -#include "serialize.h" +#include <functional> #include "sys_error.h" @@ -16,21 +16,21 @@ namespace zen // compression level must be between 0 and 9: // 0: no compression // 9: best compression -template <class BinContainer> //as specified in serialize.h -BinContainer compress(const BinContainer& stream, int level); //throw SysError +std::string compress(const std::string_view& stream, int level); //throw SysError //caveat: output stream is physically larger than input! => strip additional reserved space if needed: "BinContainer(output.begin(), output.end())" -template <class BinContainer> -BinContainer decompress(const BinContainer& stream); //throw SysError +std::string decompress(const std::string_view& stream); //throw SysError class InputStreamAsGzip //convert input stream into gzip on the fly { public: - explicit InputStreamAsGzip( //throw SysError - const std::function<size_t(void* buffer, size_t bytesToRead)>& readBlock /*throw X; returning 0 signals EOF: Posix read() semantics*/); + explicit InputStreamAsGzip(const std::function<size_t(void* buffer, size_t bytesToRead)>& tryReadBlock /*throw X; may return short, only 0 means EOF!*/, + size_t blockSize); //throw SysError ~InputStreamAsGzip(); + size_t getBlockSize() const; + size_t read(void* buffer, size_t bytesToRead); //throw SysError, X; return "bytesToRead" bytes unless end of stream! private: @@ -38,85 +38,7 @@ private: const std::unique_ptr<Impl> pimpl_; }; -std::string compressAsGzip(const void* buffer, size_t bufSize); //throw SysError - - - - - - -//######################## implementation ########################## -namespace impl -{ -size_t zlib_compressBound(size_t len); -size_t zlib_compress (const void* src, size_t srcLen, void* trg, size_t trgLen, int level); //throw SysError -size_t zlib_decompress(const void* src, size_t srcLen, void* trg, size_t trgLen); //throw SysError -} - - -template <class BinContainer> -BinContainer compress(const BinContainer& stream, int level) //throw SysError -{ - BinContainer contOut; - if (!stream.empty()) //don't dereference iterator into empty container! - { - //save uncompressed stream size for decompression - const uint64_t uncompressedSize = stream.size(); //use portable number type! - contOut.resize(sizeof(uncompressedSize)); - std::memcpy(&contOut[0], &uncompressedSize, sizeof(uncompressedSize)); - - const size_t bufferEstimate = impl::zlib_compressBound(stream.size()); //upper limit for buffer size, larger than input size!!! - - contOut.resize(contOut.size() + bufferEstimate); - - const size_t bytesWritten = impl::zlib_compress(&*stream.begin(), - stream.size(), - &*contOut.begin() + contOut.size() - bufferEstimate, - bufferEstimate, - level); //throw SysError - if (bytesWritten < bufferEstimate) - contOut.resize(contOut.size() - (bufferEstimate - bytesWritten)); //caveat: unsigned arithmetics - //caveat: physical memory consumption still *unchanged*! - } - return contOut; -} - - -template <class BinContainer> -BinContainer decompress(const BinContainer& stream) //throw SysError -{ - BinContainer contOut; - if (!stream.empty()) //don't dereference iterator into empty container! - { - //retrieve size of uncompressed data - uint64_t uncompressedSize = 0; //use portable number type! - if (stream.size() < sizeof(uncompressedSize)) - throw SysError(L"zlib error: stream size < 8"); - - std::memcpy(&uncompressedSize, &*stream.begin(), sizeof(uncompressedSize)); - - //attention: contOut MUST NOT be empty! Else it will pass a nullptr to zlib_decompress() => Z_STREAM_ERROR although "uncompressedSize == 0"!!! - //secondary bug: don't dereference iterator into empty container! - if (uncompressedSize == 0) //cannot be 0: compress() directly maps empty -> empty container skipping zlib! - throw SysError(L"zlib error: uncompressed size == 0"); - - try - { - contOut.resize(static_cast<size_t>(uncompressedSize)); //throw std::bad_alloc - } - //most likely this is due to data corruption: - catch (const std::length_error& e) { throw SysError(L"zlib error: " + _("Out of memory.") + L' ' + utfTo<std::wstring>(e.what())); } - catch (const std::bad_alloc& e) { throw SysError(L"zlib error: " + _("Out of memory.") + L' ' + utfTo<std::wstring>(e.what())); } - - const size_t bytesWritten = impl::zlib_decompress(&*stream.begin() + sizeof(uncompressedSize), - stream.size() - sizeof(uncompressedSize), - &*contOut.begin(), - static_cast<size_t>(uncompressedSize)); //throw SysError - if (bytesWritten != static_cast<size_t>(uncompressedSize)) - throw SysError(formatSystemError("zlib_decompress", L"", L"bytes written != uncompressed size.")); - } - return contOut; -} +std::string compressAsGzip(const std::string_view& stream); //throw SysError } #endif //ZLIB_WRAP_H_428597064566 diff --git a/zen/zstring.cpp b/zen/zstring.cpp index 3f5328f7..73f18cd1 100644 --- a/zen/zstring.cpp +++ b/zen/zstring.cpp @@ -11,16 +11,18 @@ using namespace zen; -Zstring getUnicodeNormalFormNonAscii(const Zstring& str) +namespace +{ +Zstring getUnicodeNormalForm_NonAsciiValidUtf(const Zstring& str, UnicodeNormalForm form) { - //Example: const char* decomposed = "\x6f\xcc\x81"; - // const char* precomposed = "\xc3\xb3"; + //Example: const char* decomposed = "\x6f\xcc\x81"; //ó + // const char* precomposed = "\xc3\xb3"; //ó assert(!isAsciiString(str)); //includes "not-empty" check assert(str.find(Zchar('\0')) == Zstring::npos); //don't expect embedded nulls! try { - gchar* outStr = ::g_utf8_normalize(str.c_str(), str.length(), G_NORMALIZE_DEFAULT_COMPOSE); + gchar* outStr = ::g_utf8_normalize(str.c_str(), str.length(), form == UnicodeNormalForm::nfc ? G_NORMALIZE_NFC : G_NORMALIZE_NFD); if (!outStr) throw SysError(formatSystemError("g_utf8_normalize", L"", L"Conversion failed.")); ZEN_ON_SCOPE_EXIT(::g_free(outStr)); @@ -29,26 +31,53 @@ Zstring getUnicodeNormalFormNonAscii(const Zstring& str) } catch (const SysError& e) { - throw std::runtime_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Error normalizing string:" + - '\n' + utfTo<std::string>(str) + "\n\n" + utfTo<std::string>(e.toString())); + throw std::runtime_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Error normalizing string:" + '\n' + + utfTo<std::string>(str) + "\n\n" + utfTo<std::string>(e.toString())); } } -Zstring getUnicodeNormalForm(const Zstring& str) +Zstring getUnicodeNormalFormNonAscii(const Zstring& str, UnicodeNormalForm form) { - //fast pre-check: - if (isAsciiString(str)) //perf: in the range of 3.5ns - return str; - static_assert(std::is_same_v<decltype(str), const Zbase<Zchar>&>, "god bless our ref-counting! => save output string memory consumption!"); + /* 1. do NOT fail on broken UTF encoding, instead normalize using REPLACEMENT_CHAR! + 2. NormalizeString() haateeez them Unicode non-characters: ERROR_NO_UNICODE_TRANSLATION! http://www.unicode.org/faq/private_use.html#nonchar1 + - No such issue on Linux/macOS with g_utf8_normalize(), and CFStringGetFileSystemRepresentation() + -> still, probably good idea to "normalize" Unicode non-characters cross-platform + - consistency for compareNoCase(): let's *unconditionally* check before other normalization operations, not just in error case! */ + using impl::CodePoint; + auto isUnicodeNonCharacter = [](CodePoint cp) { assert(cp <= impl::CODE_POINT_MAX); return (0xfdd0 <= cp && cp <= 0xfdef) || cp % 0x10'000 >= 0xfffe; }; + + const bool invalidUtf = [&] //pre-check: avoid memory allocation if valid UTF + { + UtfDecoder<Zchar> decoder(str.c_str(), str.size()); + while (const std::optional<CodePoint> cp = decoder.getNext()) + if (*cp == impl::REPLACEMENT_CHAR || //marks broken UTF encoding + isUnicodeNonCharacter(*cp)) + return true; + return false; + }(); - return getUnicodeNormalFormNonAscii(str); + if (invalidUtf) //band-aid broken UTF encoding with REPLACEMENT_CHAR + { + Zstring validStr; //don't want extra memory allocations in the standard case (valid UTF) + UtfDecoder<Zchar> decoder(str.c_str(), str.size()); + while (std::optional<CodePoint> cp = decoder.getNext()) + { + if (isUnicodeNonCharacter(*cp)) // + *cp = impl::REPLACEMENT_CHAR; //"normalize" Unicode non-characters + + codePointToUtf<Zchar>(*cp, [&](Zchar ch) { validStr += ch; }); + } + return getUnicodeNormalForm_NonAsciiValidUtf(validStr, form); + } + else + return getUnicodeNormalForm_NonAsciiValidUtf(str, form); } Zstring getUpperCaseNonAscii(const Zstring& str) { - Zstring strNorm = getUnicodeNormalFormNonAscii(str); + Zstring strNorm = getUnicodeNormalFormNonAscii(str, UnicodeNormalForm::native); try { Zstring output; @@ -64,10 +93,22 @@ Zstring getUpperCaseNonAscii(const Zstring& str) } catch (const SysError& e) { - throw std::runtime_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Error converting string to upper case:" + - '\n' + utfTo<std::string>(str) + "\n\n" + utfTo<std::string>(e.toString())); + throw std::runtime_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Error converting string to upper case:" + '\n' + + utfTo<std::string>(str) + "\n\n" + utfTo<std::string>(e.toString())); } } +} + + +Zstring getUnicodeNormalForm(const Zstring& str, UnicodeNormalForm form) +{ + //fast pre-check: + if (isAsciiString(str)) //perf: in the range of 3.5ns + return str; + static_assert(std::is_same_v<decltype(str), const Zbase<Zchar>&>, "god bless our ref-counting! => save needless memory allocation!"); + + return getUnicodeNormalFormNonAscii(str, form); +} Zstring getUpperCase(const Zstring& str) @@ -90,8 +131,8 @@ namespace std::weak_ordering compareNoCaseUtf8(const char* lhs, size_t lhsLen, const char* rhs, size_t rhsLen) { //expect Unicode normalized strings! - assert(std::string(lhs, lhsLen) == getUnicodeNormalForm(std::string(lhs, lhsLen))); - assert(std::string(rhs, rhsLen) == getUnicodeNormalForm(std::string(rhs, rhsLen))); + assert(Zstring(lhs, lhsLen) == getUnicodeNormalForm(Zstring(lhs, lhsLen), UnicodeNormalForm::nfd)); + assert(Zstring(rhs, rhsLen) == getUnicodeNormalForm(Zstring(rhs, rhsLen), UnicodeNormalForm::nfd)); //- strncasecmp implements ASCII CI-comparsion only! => signature is broken for UTF8-input; toupper() similarly doesn't support Unicode //- wcsncasecmp: https://opensource.apple.com/source/Libc/Libc-763.12/string/wcsncasecmp-fbsd.c @@ -121,14 +162,14 @@ std::weak_ordering compareNoCaseUtf8(const char* lhs, size_t lhsLen, const char* std::weak_ordering compareNatural(const Zstring& lhs, const Zstring& rhs) { - /* Unicode normal forms: - Windows: CompareString() already ignores NFD/NFC differences: nice... - Linux: g_unichar_toupper() can't ignore differences - macOS: CFStringCompare() considers differences */ try { - const Zstring& lhsNorm = getUnicodeNormalForm(lhs); - const Zstring& rhsNorm = getUnicodeNormalForm(rhs); + /* Unicode normal forms: + Windows: CompareString() ignores NFD/NFC differences and converts to NFD + Linux: g_unichar_toupper() can't ignore differences + macOS: CFStringCompare() considers differences */ + const Zstring& lhsNorm = getUnicodeNormalForm(lhs, UnicodeNormalForm::nfd); //normalize: - broken UTF encoding + const Zstring& rhsNorm = getUnicodeNormalForm(rhs, UnicodeNormalForm::nfd); // - Unicode non-characters const char* strL = lhsNorm.c_str(); const char* strR = rhsNorm.c_str(); diff --git a/zen/zstring.h b/zen/zstring.h index 692217c1..d0a8eb4c 100644 --- a/zen/zstring.h +++ b/zen/zstring.h @@ -27,7 +27,13 @@ using Zstringc = zen::Zbase<char>; //Windows, Linux: precomposed //macOS: decomposed -Zstring getUnicodeNormalForm(const Zstring& str); +enum class UnicodeNormalForm +{ + nfc, //precomposed + nfd, //decomposed + native = nfc, +}; +Zstring getUnicodeNormalForm(const Zstring& str, UnicodeNormalForm form = UnicodeNormalForm::native); /* "In fact, Unicode declares that there is an equivalence relationship between decomposed and composed sequences, and conformant software should not treat canonically equivalent sequences, whether composed or decomposed or something in between, as different." https://www.win.tue.nl/~aeb/linux/uc/nfc_vs_nfd.html */ diff --git a/zenXml/zenxml/dom.h b/zenXml/zenxml/dom.h index 427e89f2..5befdd7f 100644 --- a/zenXml/zenxml/dom.h +++ b/zenXml/zenxml/dom.h @@ -217,7 +217,7 @@ public: \code auto itPair = elem.getAttributes(); for (auto it = itPair.first; it != itPair.second; ++it) - std::cout << "name: " << it->name << " value: " << it->value << '\n'; + std::cout << std::string("name: ") + it->name + " value: " + it->value + '\n'; \endcode \return A pair of STL begin/end iterators to access all attributes sequentially as a list of name/value pairs of std::string. */ std::pair<AttrIter, AttrIter> getAttributes() const { return {attributes_.begin(), attributes_.end()}; } diff --git a/zenXml/zenxml/xml.h b/zenXml/zenxml/xml.h index 829bb5f3..c87655ba 100644 --- a/zenXml/zenxml/xml.h +++ b/zenXml/zenxml/xml.h @@ -34,16 +34,17 @@ namespace { XmlDoc loadXml(const Zstring& filePath) //throw FileError { - FileInput fileIn(filePath, nullptr /*notifyUnbufferedIO*/); //throw FileError, ErrorFileLocked - const size_t blockSize = fileIn.getBlockSize(); + FileInputPlain fileIn(filePath); //throw FileError + const size_t blockSize = fileIn.getBlockSize(); //throw FileError const std::string xmlPrefix = "<?xml version="; bool xmlPrefixChecked = false; std::string buffer; for (;;) { + warn_static("don't need zero-initialization! => resize_and_overwrite") buffer.resize(buffer.size() + blockSize); - const size_t bytesRead = fileIn.read(&*(buffer.end() - blockSize), blockSize); //throw FileError, ErrorFileLocked, (X); return "bytesToRead" bytes unless end of stream! + const size_t bytesRead = fileIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw FileError; may return short, only 0 means EOF! CONTRACT: bytesToRead > 0! buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics //quick test whether input is an XML: avoid loading large binary files up front! @@ -55,7 +56,7 @@ XmlDoc loadXml(const Zstring& filePath) //throw FileError throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filePath))); } - if (bytesRead < blockSize) //end of file + if (bytesRead == 0) //end of file break; } |