From 339ed7f63798fb5ccab05fa7fb9d0d95743c9c89 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Wed, 16 Mar 2016 21:34:59 +0100 Subject: 8.0 --- zen/dir_watcher.cpp | 4 +- zen/file_access.cpp | 641 +++++++++++++++++++++++++------------------------ zen/file_access.h | 2 +- zen/file_io.cpp | 93 ++++--- zen/file_io.h | 40 ++- zen/file_traverser.cpp | 32 +-- zen/format_unit.cpp | 4 +- zen/guid.h | 5 +- zen/i18n.h | 4 +- zen/long_path_prefix.h | 134 ----------- zen/scope_guard.h | 6 +- zen/serialize.h | 223 +++++++++++------ zen/shell_execute.h | 12 +- zen/string_base.h | 11 + zen/string_tools.h | 41 ++-- zen/thread.h | 4 +- zen/win.h | 33 --- zen/xml_io.cpp | 46 ++-- zen/zstring.cpp | 120 ++++----- 19 files changed, 697 insertions(+), 758 deletions(-) delete mode 100644 zen/long_path_prefix.h delete mode 100644 zen/win.h (limited to 'zen') diff --git a/zen/dir_watcher.cpp b/zen/dir_watcher.cpp index e7748519..702a8e32 100644 --- a/zen/dir_watcher.cpp +++ b/zen/dir_watcher.cpp @@ -358,10 +358,10 @@ std::vector DirWatcher::getChanges(const std::functionvolRemoval->requestReceived()) { - const std::chrono::steady_clock::time_point endTime = std::chrono::steady_clock::now() + std::chrono::seconds(15); + const std::chrono::steady_clock::time_point stopTime = std::chrono::steady_clock::now() + std::chrono::seconds(15); //HandleVolumeRemoval::finished() not guaranteed! note: Windows gives unresponsive applications ca. 10 seconds until unmounting the usb stick in worst case - while (!pimpl_->volRemoval->finished() && std::chrono::steady_clock::now() < endTime) + while (!pimpl_->volRemoval->finished() && std::chrono::steady_clock::now() < stopTime) { processGuiMessages(); //DBT_DEVICEREMOVECOMPLETE message is sent here! std::this_thread::sleep_for(std::chrono::milliseconds(50)); diff --git a/zen/file_access.cpp b/zen/file_access.cpp index 009d60b2..ea13d381 100644 --- a/zen/file_access.cpp +++ b/zen/file_access.cpp @@ -12,7 +12,7 @@ #include "scope_guard.h" #include "symlink_target.h" #include "file_id_def.h" -#include "serialize.h" +#include "file_io.h" #ifdef ZEN_WIN #include @@ -232,7 +232,7 @@ std::uint64_t zen::getFreeDiskSpace(const Zstring& path) //throw FileError, retu nullptr, //__out_opt PULARGE_INTEGER lpTotalNumberOfBytes, nullptr)) //__out_opt PULARGE_INTEGER lpTotalNumberOfFreeBytes THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot determine free disk space for %x."), L"%x", fmtPath(path)), L"GetDiskFreeSpaceEx"); - //succeeds even if path is not yet existing! + //succeeds even if path is not yet existing! //return 0 if info is not available: "The GetDiskFreeSpaceEx function returns zero for lpFreeBytesAvailable for all CD requests" return get64BitUInt(bytesFree.LowPart, bytesFree.HighPart); @@ -613,80 +613,191 @@ namespace { #ifdef ZEN_WIN -void setFileTimeRaw(const Zstring& filePath, - const FILETIME* creationTime, //optional - const FILETIME& lastWriteTime, - ProcSymlink procSl) //throw FileError +void setFileTimeByHandle(HANDLE hFile, //throw FileError + const FILETIME* creationTime, //optional + const FILETIME& lastWriteTime, + const Zstring& filePath) //for error message only { + if (!::SetFileTime(hFile, //__in HANDLE hFile, + creationTime, //__in_opt const FILETIME *lpCreationTime, + nullptr, //__in_opt const FILETIME *lpLastAccessTime, + &lastWriteTime)) //__in_opt const FILETIME *lpLastWriteTime { - //extra scope for debug check below - - //privilege SE_BACKUP_NAME doesn't seem to be required here for symbolic links - //note: setting privileges requires admin rights! + DWORD ec = ::GetLastError(); //copy before directly/indirectly making other system calls! - //opening newly created target file may fail due to some AV-software scanning it: no error, we will wait! - //http://support.microsoft.com/?scid=kb%3Ben-us%3B316609&x=17&y=20 - //-> enable as soon it turns out it is required! + auto toLargeInteger = [](const FILETIME& ft) -> LARGE_INTEGER + { + LARGE_INTEGER tmp = {}; + tmp.LowPart = ft.dwLowDateTime; + tmp.HighPart = ft.dwHighDateTime; + return tmp; + }; - /*const int retryInterval = 50; - const int maxRetries = 2000 / retryInterval; - for (int i = 0; i < maxRetries; ++i) +#ifdef ZEN_WIN_VISTA_AND_LATER + //function may fail if file is read-only: https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 + if (ec == ERROR_ACCESS_DENIED) { - */ + auto setFileInfo = [&](FILE_BASIC_INFO basicInfo) //throw FileError; no const& since SetFileInformationByHandle() requires non-const parameter! + { + if (!::SetFileInformationByHandle(hFile, //__in HANDLE hFile, + FileBasicInfo, //__in FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + &basicInfo, //__in LPVOID lpFileInformation, + sizeof(basicInfo))) //__in DWORD dwBufferSize + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtPath(filePath)), L"SetFileInformationByHandle"); + }; + //--------------------------------------------------------------------------- + + BY_HANDLE_FILE_INFORMATION fileInfo = {}; + if (::GetFileInformationByHandle(hFile, &fileInfo)) + if (fileInfo.dwFileAttributes & FILE_ATTRIBUTE_READONLY) + { + FILE_BASIC_INFO basicInfo = {}; //undocumented: file times of "0" stand for "don't change" + basicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL; //[!] the bug in the ticket above requires we set this together with file times!!! + basicInfo.LastWriteTime = toLargeInteger(lastWriteTime); // + if (creationTime) + basicInfo.CreationTime = toLargeInteger(*creationTime); - /* - if (hTarget == INVALID_HANDLE_VALUE && ::GetLastError() == ERROR_SHARING_VIOLATION) - ::Sleep(retryInterval); //wait then retry - else //success or unknown failure - break; + //set file time + attributes + setFileInfo(basicInfo); //throw FileError + + try //... to restore original file attributes + { + FILE_BASIC_INFO basicInfo2 = {}; + basicInfo2.FileAttributes = fileInfo.dwFileAttributes; + setFileInfo(basicInfo2); //throw FileError + } + catch (FileError&) {} + + ec = ERROR_SUCCESS; + } } - */ - //temporarily reset read-only flag if required - DWORD attribs = INVALID_FILE_ATTRIBUTES; - ZEN_ON_SCOPE_EXIT( - if (attribs != INVALID_FILE_ATTRIBUTES) - ::SetFileAttributes(applyLongPathPrefix(filePath).c_str(), attribs); - ); - - auto removeReadonly = [&]() -> bool //throw FileError; may need to remove the readonly-attribute (e.g. on FAT usb drives) +#endif + + std::wstring errorMsg = replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)); + + //add more meaningful message: FAT accepts only a subset of the NTFS date range + if (ec == ERROR_INVALID_PARAMETER && isFatDrive(filePath)) { - if (attribs == INVALID_FILE_ATTRIBUTES) + //let's not fail due to an invalid creation time on FAT: http://www.freefilesync.org/forum/viewtopic.php?t=2278 + if (creationTime) //retry (single-level recursion at most!) + return setFileTimeByHandle(hFile, nullptr, lastWriteTime, filePath); //throw FileError + + //if the ERROR_INVALID_PARAMETER is due to an invalid date, enhance message: + const LARGE_INTEGER writeTimeInt = toLargeInteger(lastWriteTime); + if (writeTimeInt.QuadPart < 0x01a8e79fe1d58000 || //1980-01-01 https://en.wikipedia.org/wiki/Time_formatting_and_storage_bugs#Year_2100 + writeTimeInt.QuadPart >= 0x022f716377640000) //2100-01-01 { - const DWORD tmpAttr = ::GetFileAttributes(applyLongPathPrefix(filePath).c_str()); - if (tmpAttr == INVALID_FILE_ATTRIBUTES) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath)), L"GetFileAttributes"); - - if (tmpAttr & FILE_ATTRIBUTE_READONLY) + errorMsg += L"\nA FAT volume can only store dates from 1980 to 2099:\n" "\twrite time (UTC):"; + SYSTEMTIME st = {}; + if (::FileTimeToSystemTime(&lastWriteTime, //__in const FILETIME *lpFileTime, + &st)) //__out LPSYSTEMTIME lpSystemTime { - if (!::SetFileAttributes(applyLongPathPrefix(filePath).c_str(), FILE_ATTRIBUTE_NORMAL)) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtPath(filePath)), L"SetFileAttributes"); + //we need a low-level reliable routine to format a potentially invalid date => don't use strftime!!! + int bufferSize = ::GetDateFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, nullptr, 0); + if (bufferSize > 0) + { + std::vector buffer(bufferSize); + if (::GetDateFormat(LOCALE_USER_DEFAULT, //_In_ LCID Locale, + 0, //_In_ DWORD dwFlags, + &st, //_In_opt_ const SYSTEMTIME *lpDate, + nullptr, //_In_opt_ LPCTSTR lpFormat, + &buffer[0], //_Out_opt_ LPTSTR lpDateStr, + bufferSize) > 0) //_In_ int cchDate + errorMsg += std::wstring(L" ") + &buffer[0]; //GetDateFormat() returns char count *including* 0-termination! + } - attribs = tmpAttr; //reapplied on scope exit - return true; + bufferSize = ::GetTimeFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, nullptr, 0); + if (bufferSize > 0) + { + std::vector buffer(bufferSize); + if (::GetTimeFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, &buffer[0], bufferSize) > 0) + errorMsg += std::wstring(L" ") + &buffer[0]; //GetDateFormat() returns char count *including* 0-termination! + } } + errorMsg += L" (" + numberTo(writeTimeInt.QuadPart) + L")"; //just in case the above date formatting fails } - return false; - }; + } + + if (ec != ERROR_SUCCESS) + throw FileError(errorMsg, formatSystemError(L"SetFileTime", ec)); + } +} + + +void setWriteTimeNative(const Zstring& filePath, + const FILETIME& lastWriteTime, + const FILETIME* creationTime, //optional + ProcSymlink procSl) //throw FileError +{ + //privilege SE_BACKUP_NAME doesn't seem to be required here for symbolic links + //note: setting privileges requires admin rights! + + //opening newly created target file may fail due to some AV-software scanning it: no error, we will wait! + //http://support.microsoft.com/?scid=kb%3Ben-us%3B316609&x=17&y=20 + //-> enable as soon it turns out it is required! + + /*const int retryInterval = 50; + const int maxRetries = 2000 / retryInterval; + for (int i = 0; i < maxRetries; ++i) + { + */ - auto openFile = [&](bool conservativeApproach) + /* + if (hTarget == INVALID_HANDLE_VALUE && ::GetLastError() == ERROR_SHARING_VIOLATION) + ::Sleep(retryInterval); //wait then retry + else //success or unknown failure + break; + } + */ + //temporarily reset read-only flag if required + DWORD attribs = INVALID_FILE_ATTRIBUTES; + ZEN_ON_SCOPE_EXIT( + if (attribs != INVALID_FILE_ATTRIBUTES) + ::SetFileAttributes(applyLongPathPrefix(filePath).c_str(), attribs); + ); + + auto removeReadonly = [&]() -> bool //throw FileError; may need to remove the readonly-attribute (e.g. on FAT usb drives) + { + if (attribs == INVALID_FILE_ATTRIBUTES) { - return ::CreateFile(applyLongPathPrefix(filePath).c_str(), //_In_ LPCTSTR lpFileName, - (conservativeApproach ? - //some NAS seem to have issues with FILE_WRITE_ATTRIBUTES, even worse, they may fail silently! - //http://sourceforge.net/tracker/?func=detail&atid=1093081&aid=3536680&group_id=234430 - //Citrix shares seem to have this issue, too, but at least fail with "access denied" => try generic access first: - GENERIC_READ | GENERIC_WRITE : - //avoids mysterious "access denied" when using "GENERIC_READ | GENERIC_WRITE" on a read-only file, even *after* read-only was removed directly before the call! - //http://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 - //since former gives an error notification we may very well try FILE_WRITE_ATTRIBUTES second. - FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES), //_In_ DWORD dwDesiredAccess, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //_In_ DWORD dwShareMode, - nullptr, //_In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes, - OPEN_EXISTING, //_In_ DWORD dwCreationDisposition, - (procSl == ProcSymlink::DIRECT ? FILE_FLAG_OPEN_REPARSE_POINT : 0) | - FILE_FLAG_BACKUP_SEMANTICS, /*needed to open a directory*/ //_In_ DWORD dwFlagsAndAttributes, - nullptr); //_In_opt_ HANDLE hTemplateFile - }; + const DWORD tmpAttr = ::GetFileAttributes(applyLongPathPrefix(filePath).c_str()); + if (tmpAttr == INVALID_FILE_ATTRIBUTES) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath)), L"GetFileAttributes"); + + if (tmpAttr & FILE_ATTRIBUTE_READONLY) + { + if (!::SetFileAttributes(applyLongPathPrefix(filePath).c_str(), FILE_ATTRIBUTE_NORMAL)) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtPath(filePath)), L"SetFileAttributes"); + + attribs = tmpAttr; //reapplied on scope exit + return true; + } + } + return false; + }; + + auto openFile = [&](bool conservativeApproach) + { + return ::CreateFile(applyLongPathPrefix(filePath).c_str(), //_In_ LPCTSTR lpFileName, + (conservativeApproach ? + //some NAS seem to have issues with FILE_WRITE_ATTRIBUTES, even worse, they may fail silently! + //http://sourceforge.net/tracker/?func=detail&atid=1093081&aid=3536680&group_id=234430 + //Citrix shares seem to have this issue, too, but at least fail with "access denied" => try generic access first: + GENERIC_READ | GENERIC_WRITE : + //avoids mysterious "access denied" when using "GENERIC_READ | GENERIC_WRITE" on a read-only file, even *after* read-only was removed directly before the call! + //http://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 + //since former gives an error notification we may very well try FILE_WRITE_ATTRIBUTES second. + FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES), //_In_ DWORD dwDesiredAccess, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //_In_ DWORD dwShareMode, + nullptr, //_In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes, + OPEN_EXISTING, //_In_ DWORD dwCreationDisposition, + (procSl == ProcSymlink::DIRECT ? FILE_FLAG_OPEN_REPARSE_POINT : 0) | + FILE_FLAG_BACKUP_SEMANTICS, /*needed to open a directory*/ //_In_ DWORD dwFlagsAndAttributes, + nullptr); //_In_opt_ HANDLE hTemplateFile + }; + + { + //extra scope for debug check below HANDLE hFile = INVALID_HANDLE_VALUE; for (int i = 0; i < 2; ++i) //we will get this handle, no matter what! :) @@ -716,118 +827,7 @@ void setFileTimeRaw(const Zstring& filePath, } ZEN_ON_SCOPE_EXIT(::CloseHandle(hFile)); - - if (!::SetFileTime(hFile, //__in HANDLE hFile, - creationTime, //__in_opt const FILETIME *lpCreationTime, - nullptr, //__in_opt const FILETIME *lpLastAccessTime, - &lastWriteTime)) //__in_opt const FILETIME *lpLastWriteTime - { - DWORD ec = ::GetLastError(); //copy before directly/indirectly making other system calls! - - //function may fail if file is read-only: https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 - if (ec == ERROR_ACCESS_DENIED) - { - //dynamically load windows API function: available with Windows Vista and later - typedef BOOL (WINAPI* SetFileInformationByHandleFunc)(HANDLE hFile, FILE_INFO_BY_HANDLE_CLASS FileInformationClass, LPVOID lpFileInformation, DWORD dwBufferSize); - const SysDllFun setFileInformationByHandle(L"kernel32.dll", "SetFileInformationByHandle"); - - if (setFileInformationByHandle) //if not: let the original error propagate! - { - auto setFileInfo = [&](FILE_BASIC_INFO basicInfo) //throw FileError; no const& since SetFileInformationByHandle() requires non-const parameter! - { - if (!setFileInformationByHandle(hFile, //__in HANDLE hFile, - FileBasicInfo, //__in FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - &basicInfo, //__in LPVOID lpFileInformation, - sizeof(basicInfo))) //__in DWORD dwBufferSize - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtPath(filePath)), L"SetFileInformationByHandle"); - }; - - auto toLargeInteger = [](const FILETIME& ft) -> LARGE_INTEGER - { - LARGE_INTEGER tmp = {}; - tmp.LowPart = ft.dwLowDateTime; - tmp.HighPart = ft.dwHighDateTime; - return tmp; - }; - //--------------------------------------------------------------------------- - - BY_HANDLE_FILE_INFORMATION fileInfo = {}; - if (::GetFileInformationByHandle(hFile, &fileInfo)) - if (fileInfo.dwFileAttributes & FILE_ATTRIBUTE_READONLY) - { - FILE_BASIC_INFO basicInfo = {}; //undocumented: file times of "0" stand for "don't change" - basicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL; //[!] the bug in the ticket above requires we set this together with file times!!! - basicInfo.LastWriteTime = toLargeInteger(lastWriteTime); // - if (creationTime) - basicInfo.CreationTime = toLargeInteger(*creationTime); - - //set file time + attributes - setFileInfo(basicInfo); //throw FileError - - try //... to restore original file attributes - { - FILE_BASIC_INFO basicInfo2 = {}; - basicInfo2.FileAttributes = fileInfo.dwFileAttributes; - setFileInfo(basicInfo2); //throw FileError - } - catch (FileError&) {} - - ec = ERROR_SUCCESS; - } - } - } - - std::wstring errorMsg = replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)); - - //add more meaningful message: FAT accepts only a subset of the NTFS date range - if (ec == ERROR_INVALID_PARAMETER && - isFatDrive(filePath)) - { - //we need a low-level reliable routine to format a potentially invalid date => don't use strftime!!! - auto fmtDate = [](const FILETIME& ft) - { - SYSTEMTIME st = {}; - if (!::FileTimeToSystemTime(&ft, //__in const FILETIME *lpFileTime, - &st)) //__out LPSYSTEMTIME lpSystemTime - return std::wstring(); - - std::wstring dateTime; - { - const int bufferSize = ::GetDateFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, nullptr, 0); - if (bufferSize > 0) - { - std::vector buffer(bufferSize); - if (::GetDateFormat(LOCALE_USER_DEFAULT, //_In_ LCID Locale, - 0, //_In_ DWORD dwFlags, - &st, //_In_opt_ const SYSTEMTIME *lpDate, - nullptr, //_In_opt_ LPCTSTR lpFormat, - &buffer[0], //_Out_opt_ LPTSTR lpDateStr, - bufferSize) > 0) //_In_ int cchDate - dateTime = &buffer[0]; //GetDateFormat() returns char count *including* 0-termination! - } - } - - const int bufferSize = ::GetTimeFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, nullptr, 0); - if (bufferSize > 0) - { - std::vector buffer(bufferSize); - if (::GetTimeFormat(LOCALE_USER_DEFAULT, 0, &st, nullptr, &buffer[0], bufferSize) > 0) - { - dateTime += L" "; - dateTime += &buffer[0]; //GetDateFormat() returns char count *including* 0-termination! - } - } - return dateTime; - }; - - errorMsg += std::wstring(L"\nA FAT volume can only store dates between 1980 and 2107:\n") + - L"\twrite (UTC): \t" + fmtDate(lastWriteTime) + - (creationTime ? L"\n\tcreate (UTC): \t" + fmtDate(*creationTime) : L""); - } - - if (ec != ERROR_SUCCESS) - throw FileError(errorMsg, formatSystemError(L"SetFileTime", ec)); - } + setFileTimeByHandle(hFile, creationTime, lastWriteTime, filePath); //throw FileError } #ifndef NDEBUG //verify written data FILETIME creationTimeDbg = {}; @@ -856,9 +856,26 @@ void setFileTimeRaw(const Zstring& filePath, #elif defined ZEN_LINUX -DEFINE_NEW_FILE_ERROR(ErrorLinuxFallbackToUtimes); +void setWriteTimeFallback(const Zstring& filePath, std::int64_t modTime, ProcSymlink procSl) //throw FileError +{ + struct ::timeval writeTime[2] = {}; + writeTime[0].tv_sec = ::time(nullptr); //access time (seconds) + writeTime[1].tv_sec = modTime; //modification time (seconds) + + if (procSl == ProcSymlink::FOLLOW) + { + if (::utimes(filePath.c_str(), writeTime) != 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"utimes"); + } + else + { + if (::lutimes(filePath.c_str(), writeTime) != 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"lutimes"); + } +} -void setFileTimeRaw(const Zstring& filePath, const struct ::timespec& modTime, ProcSymlink procSl) //throw FileError, ErrorLinuxFallbackToUtimes + +void setWriteTimeNative(const Zstring& filePath, std::int64_t modTime, ProcSymlink procSl) //throw FileError { /* [2013-05-01] sigh, we can't use utimensat() on NTFS volumes on Ubuntu: silent failure!!! what morons are programming this shit??? @@ -867,15 +884,15 @@ void setFileTimeRaw(const Zstring& filePath, const struct ::timespec& modTime, P [2015-03-09] - cannot reproduce issues with NTFS and utimensat() on Ubuntu - utimensat() is supposed to obsolete utime/utimes and is also used by "cp" and "touch" - - solves utimes() EINVAL bug for certain CIFS/NTFS drives: https://sourceforge.net/p/freefilesync/discussion/help/thread/1ace042d/ + - solves utimes() EINVAL bug for certain CIFS/NTFS drives: http://www.freefilesync.org/forum/viewtopic.php?t=387 => don't use utimensat() directly, but open file descriptor manually, else EINVAL, again! => let's give utimensat another chance: */ struct ::timespec newTimes[2] = {}; newTimes[0].tv_sec = ::time(nullptr); //access time; using UTIME_OMIT for tv_nsec would trigger even more bugs!! - //https://sourceforge.net/p/freefilesync/discussion/open-discussion/thread/218564cf/ - newTimes[1] = modTime; //modification time + //http://www.freefilesync.org/forum/viewtopic.php?t=1701 + newTimes[1].tv_sec = modTime; //modification time //=> using open()/futimens() for regular files and utimensat(AT_SYMLINK_NOFOLLOW) for symlinks is consistent with "cp" and "touch"! if (procSl == ProcSymlink::FOLLOW) @@ -884,7 +901,7 @@ void setFileTimeRaw(const Zstring& filePath, const struct ::timespec& modTime, P if (fdFile == -1) { if (errno == EACCES) //bullshit, access denied even with 0777 permissions! => utimes should work! - throw ErrorLinuxFallbackToUtimes(L""); + return setWriteTimeFallback(filePath, modTime, procSl); //throw FileError THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"open"); } @@ -910,10 +927,10 @@ struct AttrBufFileTimes } __attribute__((aligned(4), packed)); -void setFileTimeRaw(const Zstring& filePath, - const struct ::timespec* createTime, //optional - const struct ::timespec& writeTime, - ProcSymlink procSl) //throw FileError +void setWriteTimeNative(const Zstring& filePath, + const struct ::timespec& writeTime, + const struct ::timespec* createTime, //optional + ProcSymlink procSl) //throw FileError { //OS X: utime() is obsoleted by utimes()! utimensat() not yet implemented //use ::setattrlist() instead of ::utimes() => 1. set file creation times 2. nanosecond precision @@ -923,19 +940,19 @@ void setFileTimeRaw(const Zstring& filePath, attribs.bitmapcount = ATTR_BIT_MAP_COUNT; attribs.commonattr = (createTime ? ATTR_CMN_CRTIME : 0) | ATTR_CMN_MODTIME; - AttrBufFileTimes newTimes; + AttrBufFileTimes attrBuf; if (createTime) { - newTimes.createTime.tv_sec = createTime->tv_sec; - newTimes.createTime.tv_nsec = createTime->tv_nsec; + attrBuf.createTime.tv_sec = createTime->tv_sec; + attrBuf.createTime.tv_nsec = createTime->tv_nsec; } - newTimes.writeTime.tv_sec = writeTime.tv_sec; - newTimes.writeTime.tv_nsec = writeTime.tv_nsec; + attrBuf.writeTime.tv_sec = writeTime.tv_sec; + attrBuf.writeTime.tv_nsec = writeTime.tv_nsec; const int rv = ::setattrlist(filePath.c_str(), //const char* path, &attribs, //struct ::attrlist* attrList, - createTime ? &newTimes.createTime : &newTimes.writeTime, //void* attrBuf, - (createTime ? sizeof(newTimes.createTime) : 0) + sizeof(newTimes.writeTime), //size_t attrBufSize, + createTime ? &attrBuf.createTime : &attrBuf.writeTime, //void* attrBuf, + (createTime ? sizeof(attrBuf.createTime) : 0) + sizeof(attrBuf.writeTime), //size_t attrBufSize, procSl == ProcSymlink::DIRECT ? FSOPT_NOFOLLOW : 0); //unsigned long options if (rv != 0) THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"setattrlist"); @@ -952,22 +969,52 @@ void getFileTimeRaw(int fd, //throw FileError attribs.bitmapcount = ATTR_BIT_MAP_COUNT; attribs.commonattr = ATTR_CMN_CRTIME | ATTR_CMN_MODTIME; - AttrBufFileTimes fileTimes; + AttrBufFileTimes attrBuf; - const int rv = ::fgetattrlist(fd, //int fd, - &attribs, //struct ::attrlist* attrList, - &fileTimes, //void* attrBuf, - sizeof(fileTimes), //size_t attrBufSize, - 0); //unsigned long options + const int rv = ::fgetattrlist(fd, //int fd, + &attribs, //struct ::attrlist* attrList, + &attrBuf, //void* attrBuf, + sizeof(attrBuf), //size_t attrBufSize, + 0); //unsigned long options if (rv != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath)), L"getattrlist"); + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath)), L"fgetattrlist"); - createTime.tv_sec = fileTimes.createTime.tv_sec; - createTime.tv_nsec = fileTimes.createTime.tv_nsec; - writeTime.tv_sec = fileTimes.writeTime.tv_sec; - writeTime.tv_nsec = fileTimes.writeTime.tv_nsec; + createTime.tv_sec = attrBuf.createTime.tv_sec; + createTime.tv_nsec = attrBuf.createTime.tv_nsec; + writeTime.tv_sec = attrBuf.writeTime.tv_sec; + writeTime.tv_nsec = attrBuf.writeTime.tv_nsec; } */ + +//PERF: ~10µs per call for "int fd" variant (irrespective if file is local or on mounted USB; test case: 3000 files) +bool hasNativeSupportForExtendedAtrributes(const Zstring& filePath, //throw FileError + int fd = -1) //speed up!? +{ + struct ::statfs volInfo = {}; + if ((fd != -1 ? ::fstatfs(fd, &volInfo) : ::statfs(filePath.c_str(), &volInfo)) != 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(filePath)), fd != -1 ? L"fstatfs" : L"statfs"); + + struct ::attrlist attribs = {}; + attribs.bitmapcount = ATTR_BIT_MAP_COUNT; + attribs.volattr = ATTR_VOL_INFO | ATTR_VOL_CAPABILITIES; + + struct + { + std::uint32_t length = 0; + vol_capabilities_attr_t volCaps{}; + } __attribute__((aligned(4), packed)) attrBuf; + + const int rv = ::getattrlist(volInfo.f_mntonname, //const char* path, + &attribs, //struct ::attrlist* attrList, + &attrBuf, //void* attrBuf, + sizeof(attrBuf), //size_t attrBufSize, + 0); //unsigned long options + if (rv != 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(volInfo.f_mntonname)), L"getattrlist"); + + return (attrBuf.volCaps.valid [VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR) != 0 && + (attrBuf.volCaps.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR) != 0; +} #endif } @@ -975,37 +1022,15 @@ void getFileTimeRaw(int fd, //throw FileError void zen::setFileTime(const Zstring& filePath, std::int64_t modTime, ProcSymlink procSl) //throw FileError { #ifdef ZEN_WIN - setFileTimeRaw(filePath, nullptr, timetToFileTime(modTime), procSl); //throw FileError + setWriteTimeNative(filePath, timetToFileTime(modTime), nullptr, procSl); //throw FileError #elif defined ZEN_LINUX - try - { - struct ::timespec writeTime = {}; - writeTime.tv_sec = modTime; - setFileTimeRaw(filePath, writeTime, procSl); //throw FileError, ErrorLinuxFallbackToUtimes - } - catch (ErrorLinuxFallbackToUtimes&) - { - struct ::timeval writeTime[2] = {}; - writeTime[0].tv_sec = ::time(nullptr); //access time (seconds) - writeTime[1].tv_sec = modTime; //modification time (seconds) - - if (procSl == ProcSymlink::FOLLOW) - { - if (::utimes(filePath.c_str(), writeTime) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"utimes"); - } - else - { - if (::lutimes(filePath.c_str(), writeTime) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(filePath)), L"lutimes"); - } - } + setWriteTimeNative(filePath, modTime, procSl); //throw FileError #elif defined ZEN_MAC struct ::timespec writeTime = {}; writeTime.tv_sec = modTime; - setFileTimeRaw(filePath, nullptr, writeTime, procSl); //throw FileError + setWriteTimeNative(filePath, writeTime, nullptr, procSl); //throw FileError #endif } @@ -1351,20 +1376,16 @@ void zen::copyNewDirectory(const Zstring& sourcePath, const Zstring& targetPath, auto getErrorMsg = [](const Zstring& path) { return replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(path)); }; //special handling for volume root: trying to create existing root directory results in ERROR_ACCESS_DENIED rather than ERROR_ALREADY_EXISTS! - Zstring dirTmp = removeLongPathPrefix(endsWith(targetPath, FILE_NAME_SEPARATOR) ? - beforeLast(targetPath, FILE_NAME_SEPARATOR, IF_MISSING_RETURN_NONE) : - targetPath); - if (dirTmp.size() == 2 && - isAlpha(dirTmp[0]) && dirTmp[1] == L':') + const Zstring pathPf = appendSeparator(targetPath); //we do not support "C:" to represent a relative path! + if (pathPf.size() == 3 && + isAlpha(pathPf[0]) && pathPf[1] == L':') { - dirTmp += FILE_NAME_SEPARATOR; //we do not support "C:" to represent a relative path! - - const DWORD ec = somethingExists(dirTmp) ? ERROR_ALREADY_EXISTS : ERROR_PATH_NOT_FOUND; //don't use dirExists() => harmonize with ErrorTargetExisting! + const DWORD ec = somethingExists(pathPf) ? ERROR_ALREADY_EXISTS : ERROR_PATH_NOT_FOUND; //don't use dirExists() => harmonize with ErrorTargetExisting! const std::wstring errorDescr = formatSystemError(L"CreateDirectory", ec); if (ec == ERROR_ALREADY_EXISTS) - throw ErrorTargetExisting(getErrorMsg(dirTmp), errorDescr); - throw FileError(getErrorMsg(dirTmp), errorDescr); //[!] this is NOT a ErrorTargetPathMissing case! + throw ErrorTargetExisting(getErrorMsg(pathPf), errorDescr); + throw FileError(getErrorMsg(pathPf), errorDescr); //[!] this is NOT a ErrorTargetPathMissing case! } //deliberately don't support creating irregular folders like "...." https://social.technet.microsoft.com/Forums/windows/en-US/ffee2322-bb6b-4fdf-86f9-8f93cf1fa6cb/ @@ -1498,7 +1519,8 @@ void zen::copyNewDirectory(const Zstring& sourcePath, const Zstring& targetPath, } #elif defined ZEN_MAC - /*int rv =*/ ::copyfile(sourcePath.c_str(), targetPath.c_str(), nullptr, COPYFILE_XATTR); + if (hasNativeSupportForExtendedAtrributes(targetPath)) //throw FileError + ::copyfile(sourcePath.c_str(), targetPath.c_str(), nullptr, COPYFILE_XATTR); //ignore errors, see related comments in copyFileOsSpecific() #endif ZEN_ON_SCOPE_FAIL(try { removeDirectorySimple(targetPath); } @@ -1564,24 +1586,24 @@ void zen::copySymlink(const Zstring& sourceLink, const Zstring& targetLink, bool &sourceAttr)) //__out LPVOID lpFileInformation THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(sourceLink)), L"GetFileAttributesEx"); - setFileTimeRaw(targetLink, &sourceAttr.ftCreationTime, sourceAttr.ftLastWriteTime, ProcSymlink::DIRECT); //throw FileError + setWriteTimeNative(targetLink, sourceAttr.ftLastWriteTime, &sourceAttr.ftCreationTime, ProcSymlink::DIRECT); //throw FileError #elif defined ZEN_LINUX struct ::stat sourceInfo = {}; if (::lstat(sourceLink.c_str(), &sourceInfo) != 0) THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(sourceLink)), L"lstat"); - setFileTime(targetLink, sourceInfo.st_mtime, ProcSymlink::DIRECT); //throw FileError + setWriteTimeNative(targetLink, sourceInfo.st_mtime, ProcSymlink::DIRECT); //throw FileError #elif defined ZEN_MAC struct ::stat sourceInfo = {}; if (::lstat(sourceLink.c_str(), &sourceInfo) != 0) THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(sourceLink)), L"lstat"); - if (::copyfile(sourceLink.c_str(), targetLink.c_str(), nullptr, COPYFILE_XATTR | COPYFILE_NOFOLLOW) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(replaceCpy(_("Cannot copy attributes from %x to %y."), L"%x", L"\n" + fmtPath(sourceLink)), L"%y", L"\n" + fmtPath(targetLink)), L"copyfile"); + if (hasNativeSupportForExtendedAtrributes(targetLink)) //throw FileError + ::copyfile(sourceLink.c_str(), targetLink.c_str(), nullptr, COPYFILE_XATTR | COPYFILE_NOFOLLOW); //ignore errors, see related comments in copyFileOsSpecific() - setFileTimeRaw(targetLink, &sourceInfo.st_birthtimespec, sourceInfo.st_mtimespec, ProcSymlink::DIRECT); //throw FileError + setWriteTimeNative(targetLink, sourceInfo.st_mtimespec, &sourceInfo.st_birthtimespec, ProcSymlink::DIRECT); //throw FileError #endif if (copyFilePermissions) @@ -1720,10 +1742,10 @@ bool canCopyAsSparse(const Zstring& sourceFile, const Zstring& targetFile) //thr InSyncAttributes copyFileWindowsBackupStream(const Zstring& sourceFile, //throw FileError, ErrorTargetExisting, ErrorFileLocked const Zstring& targetFile, - const std::function& onUpdateCopyStatus) + const std::function& notifyProgress) { //try to get backup read and write privileges: help solve most "access denied" errors with FILE_FLAG_BACKUP_SEMANTICS: - //https://sourceforge.net/p/freefilesync/discussion/open-discussion/thread/1998ebf2/ + //http://www.freefilesync.org/forum/viewtopic.php?t=1714 try { activatePrivilege(SE_BACKUP_NAME); } catch (const FileError&) {} try { activatePrivilege(SE_RESTORE_NAME); } @@ -1933,7 +1955,7 @@ InSyncAttributes copyFileWindowsBackupStream(const Zstring& sourceFile, //throw throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(targetFile)), L"BackupWrite: incomplete write."); //user should never see this //total bytes transferred may be larger than file size! context information + ADS or smaller (sparse, compressed)! - if (onUpdateCopyStatus) onUpdateCopyStatus(bytesRead); //throw X! + if (notifyProgress) notifyProgress(bytesRead); //throw X! if (bytesRead > 0) someBytesRead = true; @@ -1946,11 +1968,7 @@ InSyncAttributes copyFileWindowsBackupStream(const Zstring& sourceFile, //throw throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(sourceFile)), L"BackupRead: unknown error"); //user should never see this -> this method is called only if "canCopyAsSparse()" //time needs to be set at the end: BackupWrite() changes modification time - if (!::SetFileTime(hFileTarget, - &fileInfoSource.ftCreationTime, - nullptr, - &fileInfoSource.ftLastWriteTime)) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(targetFile)), L"SetFileTime"); + setFileTimeByHandle(hFileTarget, &fileInfoSource.ftCreationTime,fileInfoSource.ftLastWriteTime, targetFile); //throw FileError return newAttrib; } @@ -1961,16 +1979,16 @@ DEFINE_NEW_FILE_ERROR(ErrorFallbackToCopyAsBackupStream); struct CallbackData { - CallbackData(const std::function& onUpdateCopyStatus, + CallbackData(const std::function& notifyProgress, const Zstring& sourceFile, const Zstring& targetFile) : sourceFile_(sourceFile), targetFile_(targetFile), - onUpdateCopyStatus_(onUpdateCopyStatus) {} + notifyProgress_(notifyProgress) {} const Zstring& sourceFile_; const Zstring& targetFile_; - const std::function& onUpdateCopyStatus_; //optional + const std::function& notifyProgress_; //optional std::exception_ptr exception; //out BY_HANDLE_FILE_INFORMATION fileInfoSrc{}; //out: modified by CopyFileEx() at beginning @@ -2059,9 +2077,9 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, } } - if (cbd.onUpdateCopyStatus_ && totalBytesTransferred.QuadPart >= 0) //should always be true, but let's still check + if (cbd.notifyProgress_ && totalBytesTransferred.QuadPart >= 0) //should always be true, but let's still check { - cbd.onUpdateCopyStatus_(totalBytesTransferred.QuadPart - cbd.bytesReported); //throw X! + cbd.notifyProgress_(totalBytesTransferred.QuadPart - cbd.bytesReported); //throw X! cbd.bytesReported = totalBytesTransferred.QuadPart; } } @@ -2076,7 +2094,7 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, InSyncAttributes copyFileWindowsDefault(const Zstring& sourceFile, //throw FileError, ErrorTargetExisting, ErrorFileLocked, ErrorFallbackToCopyAsBackupStream const Zstring& targetFile, - const std::function& onUpdateCopyStatus) + const std::function& notifyProgress) { //try to get backup read and write privileges: may help solve some "access denied" errors bool backupPrivilegesActive = true; @@ -2095,12 +2113,12 @@ InSyncAttributes copyFileWindowsDefault(const Zstring& sourceFile, //throw FileE //if (vistaOrLater()) //see http://blogs.technet.com/b/askperf/archive/2007/05/08/slow-large-file-copy-issues.aspx // copyFlags |= COPY_FILE_NO_BUFFERING; //no perf difference at worst, improvement for large files (20% in test NTFS -> NTFS) - // - this flag may cause file corruption! https://sourceforge.net/p/freefilesync/discussion/open-discussion/thread/65f48357/ + // - this flag may cause file corruption! http://www.freefilesync.org/forum/viewtopic.php?t=1857 // - documentation on CopyFile2() even states: "It is not recommended to pause copies that are using this flag." //=> it's not worth it! instead of skipping buffering at kernel-level (=> also NO prefetching!!!), skip it at user-level: memory mapped files! // however, perf-measurements for memory mapped files show: it's also not worth it! - CallbackData cbd(onUpdateCopyStatus, sourceFile, targetFile); + CallbackData cbd(notifyProgress, sourceFile, targetFile); const bool success = ::CopyFileEx(applyLongPathPrefix(sourceFile).c_str(), //__in LPCTSTR lpExistingFileName, applyLongPathPrefix(targetFile).c_str(), //__in LPCTSTR lpNewFileName, @@ -2122,10 +2140,10 @@ InSyncAttributes copyFileWindowsDefault(const Zstring& sourceFile, //throw FileE throw ErrorFallbackToCopyAsBackupStream(L"sparse, copy failure"); if (ec == ERROR_ACCESS_DENIED && backupPrivilegesActive) - //chances are good this will work with copyFileWindowsBackupStream: https://sourceforge.net/p/freefilesync/discussion/open-discussion/thread/1998ebf2/ + //chances are good this will work with copyFileWindowsBackupStream: http://www.freefilesync.org/forum/viewtopic.php?t=1714 throw ErrorFallbackToCopyAsBackupStream(L"access denied"); - //copying ADS may incorrectly fail with ERROR_FILE_NOT_FOUND: https://sourceforge.net/p/freefilesync/discussion/help/thread/a18a2c02/ + //copying ADS may incorrectly fail with ERROR_FILE_NOT_FOUND: http://www.freefilesync.org/forum/viewtopic.php?t=446 if (ec == ERROR_FILE_NOT_FOUND && cbd.fileInfoSrc.nNumberOfLinks > 0 && cbd.fileInfoTrg.nNumberOfLinks > 0) @@ -2175,7 +2193,7 @@ InSyncAttributes copyFileWindowsDefault(const Zstring& sourceFile, //throw FileE //caveat: - ::CopyFileEx() silently *ignores* failure to set modification time!!! => we always need to set it again but with proper error checking! // - perf: recent measurements show no slow down at all for buffered USB sticks! - setFileTimeRaw(targetFile, &cbd.fileInfoSrc.ftCreationTime, cbd.fileInfoSrc.ftLastWriteTime, ProcSymlink::FOLLOW); //throw FileError + setWriteTimeNative(targetFile, cbd.fileInfoSrc.ftLastWriteTime, &cbd.fileInfoSrc.ftCreationTime, ProcSymlink::FOLLOW); //throw FileError InSyncAttributes newAttrib; newAttrib.fileSize = get64BitUInt(cbd.fileInfoSrc.nFileSizeLow, cbd.fileInfoSrc.nFileSizeHigh); @@ -2188,15 +2206,15 @@ InSyncAttributes copyFileWindowsDefault(const Zstring& sourceFile, //throw FileE //another layer to support copying sparse files and handle some access denied errors inline -InSyncAttributes copyFileWindowsSelectRoutine(const Zstring& sourceFile, const Zstring& targetFile, const std::function& onUpdateCopyStatus) +InSyncAttributes copyFileWindowsSelectRoutine(const Zstring& sourceFile, const Zstring& targetFile, const std::function& notifyProgress) { try { - return copyFileWindowsDefault(sourceFile, targetFile, onUpdateCopyStatus); //throw FileError, ErrorTargetExisting, ErrorFileLocked, ErrorFallbackToCopyAsBackupStream + return copyFileWindowsDefault(sourceFile, targetFile, notifyProgress); //throw FileError, ErrorTargetExisting, ErrorFileLocked, ErrorFallbackToCopyAsBackupStream } catch (ErrorFallbackToCopyAsBackupStream&) { - return copyFileWindowsBackupStream(sourceFile, targetFile, onUpdateCopyStatus); //throw FileError, ErrorTargetExisting, ErrorFileLocked + return copyFileWindowsBackupStream(sourceFile, targetFile, notifyProgress); //throw FileError, ErrorTargetExisting, ErrorFileLocked } } @@ -2205,11 +2223,11 @@ InSyncAttributes copyFileWindowsSelectRoutine(const Zstring& sourceFile, const Z inline InSyncAttributes copyFileOsSpecific(const Zstring& sourceFile, const Zstring& targetFile, - const std::function& onUpdateCopyStatus) + const std::function& notifyProgress) { try { - return copyFileWindowsSelectRoutine(sourceFile, targetFile, onUpdateCopyStatus); //throw FileError, ErrorTargetExisting, ErrorFileLocked + return copyFileWindowsSelectRoutine(sourceFile, targetFile, notifyProgress); //throw FileError, ErrorTargetExisting, ErrorFileLocked } catch (const ErrorTargetExisting&) { @@ -2217,7 +2235,7 @@ InSyncAttributes copyFileOsSpecific(const Zstring& sourceFile, if (have8dot3NameClash(targetFile)) { Fix8Dot3NameClash dummy(targetFile); //throw FileError; move clashing file path to the side - return copyFileWindowsSelectRoutine(sourceFile, targetFile, onUpdateCopyStatus); //throw FileError; the short file path name clash is solved, this should work now + return copyFileWindowsSelectRoutine(sourceFile, targetFile, notifyProgress); //throw FileError; the short file path name clash is solved, this should work now } throw; } @@ -2227,19 +2245,20 @@ InSyncAttributes copyFileOsSpecific(const Zstring& sourceFile, #elif defined ZEN_LINUX || defined ZEN_MAC InSyncAttributes copyFileOsSpecific(const Zstring& sourceFile, //throw FileError, ErrorTargetExisting const Zstring& targetFile, - const std::function& onUpdateCopyStatus) + const std::function& notifyProgress) { FileInput fileIn(sourceFile); //throw FileError + if (notifyProgress) notifyProgress(0); //throw X! struct ::stat sourceInfo = {}; if (::fstat(fileIn.getHandle(), &sourceInfo) != 0) THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(sourceFile)), L"fstat"); - + const mode_t mode = sourceInfo.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); //analog to "cp" which copies "mode" (considering umask) by default - //it seems we don't need S_IWUSR, not even for the setFileTime() below! (tested with source file having different user/group!) - - const int fdTarget = ::open(targetFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + //it seems we don't need S_IWUSR, not even for the setFileTime() below! (tested with source file having different user/group!) + //=> need copyItemPermissions() only for "chown" and umask-agnostic permissions + const int fdTarget = ::open(targetFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); if (fdTarget == -1) { const int ec = errno; //copy before making other system calls! @@ -2251,66 +2270,62 @@ InSyncAttributes copyFileOsSpecific(const Zstring& sourceFile, //throw FileError throw FileError(errorMsg, errorDescr); } - if (onUpdateCopyStatus) onUpdateCopyStatus(0); //throw X! - - InSyncAttributes newAttrib; ZEN_ON_SCOPE_FAIL( try { removeFile(targetFile); } catch (FileError&) {} ); //transactional behavior: place guard after ::open() and before lifetime of FileOutput: //=> don't delete file that existed previously!!! - { - FileOutput fileOut(fdTarget, targetFile); //pass ownership - if (onUpdateCopyStatus) onUpdateCopyStatus(0); //throw X! - - copyStream(fileIn, fileOut, std::min(fileIn .optimalBlockSize(), - fileOut.optimalBlockSize()), onUpdateCopyStatus); //throw FileError, X - - struct ::stat targetInfo = {}; - if (::fstat(fileOut.getHandle(), &targetInfo) != 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(targetFile)), L"fstat"); + FileOutput fileOut(fdTarget, targetFile); //pass ownership + if (notifyProgress) notifyProgress(0); //throw X! - newAttrib.fileSize = sourceInfo.st_size; -#ifdef ZEN_MAC - newAttrib.modificationTime = sourceInfo.st_mtimespec.tv_sec; //use same time variable like setFileTimeRaw() for consistency -#else - newAttrib.modificationTime = sourceInfo.st_mtime; -#endif - newAttrib.sourceFileId = extractFileId(sourceInfo); - newAttrib.targetFileId = extractFileId(targetInfo); + unbufferedStreamCopy(fileIn, fileOut, notifyProgress); //throw FileError, X #ifdef ZEN_MAC - //using ::copyfile with COPYFILE_DATA seems to trigger bugs unlike our stream-based copying! - //=> use ::copyfile for extended attributes only: https://sourceforge.net/p/freefilesync/discussion/help/thread/91384c8a/ - //http://blog.plasticsfuture.org/2006/03/05/the-state-of-backup-and-cloning-tools-under-mac-os-x/ - //docs: http://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/copyfile.3.html - //source: http://www.opensource.apple.com/source/copyfile/copyfile-103.92.1/copyfile.c + //using ::copyfile with COPYFILE_DATA seems to trigger bugs unlike our stream-based copying! + //=> use ::copyfile for extended attributes only: http://www.freefilesync.org/forum/viewtopic.php?t=401 + //http://blog.plasticsfuture.org/2006/03/05/the-state-of-backup-and-cloning-tools-under-mac-os-x/ + //docs: http://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/copyfile.3.html + //source: http://www.opensource.apple.com/source/copyfile/copyfile-103.92.1/copyfile.c + + //avoid creation of ._ files if target doesn't support extended attributes: http://www.freefilesync.org/forum/viewtopic.php?t=2226 + if (hasNativeSupportForExtendedAtrributes(targetFile, fileOut.getHandle())) //throw FileError if (::fcopyfile(fileIn.getHandle(), fileOut.getHandle(), nullptr, COPYFILE_XATTR) != 0) - { - /* - extended attributes are only optional here => ignore error - E2BIG - reference email: "FFS V7.8 on Mac with 10.11.2 ElCapitan" - EINVAL - reference email: "Error Code 22: Invalid argument (copyfile)" - */ - //THROW_LAST_FILE_ERROR(replaceCpy(replaceCpy(_("Cannot copy attributes from %x to %y."), L"%x", L"\n" + fmtPath(sourceFile)), L"%y", L"\n" + fmtPath(targetFile)), L"fcopyfile"); - } + ; //THROW_LAST_FILE_ERROR(replaceCpy(replaceCpy(_("Cannot copy attributes from %x to %y."), L"%x", L"\n" + fmtPath(sourceFile)), L"%y", L"\n" + fmtPath(targetFile)), L"fcopyfile"); + /* + both problems still occur even with the "hasNativeSupportForExtendedAtrributes" check above: + E2BIG - reference email: "Re: FFS V7.8 on Mac with 10.11.2 ElCapitan" + EINVAL - reference email: "Error Code 22: Invalid argument (copyfile)" + */ #endif + struct ::stat targetInfo = {}; + if (::fstat(fileOut.getHandle(), &targetInfo) != 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(targetFile)), L"fstat"); - fileOut.close(); //throw FileError -> optional, but good place to catch errors when closing stream! - } //close output file handle before setting file time + //close output file handle before setting file time; also good place to catch errors when closing stream! + fileOut.close(); //throw FileError + if (notifyProgress) notifyProgress(0); //throw X! //we cannot set the target file times (::futimes) while the file descriptor is still open after a write operation: //this triggers bugs on samba shares where the modification time is set to current time instead. //Linux: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=340236 // http://comments.gmane.org/gmane.linux.file-systems.cifs/2854 - //OS X: https://sourceforge.net/p/freefilesync/discussion/help/thread/881357c0/ + //OS X: http://www.freefilesync.org/forum/viewtopic.php?t=356 #ifdef ZEN_MAC - setFileTimeRaw(targetFile, &sourceInfo.st_birthtimespec, sourceInfo.st_mtimespec, ProcSymlink::FOLLOW); //throw FileError + setWriteTimeNative(targetFile, sourceInfo.st_mtimespec, &sourceInfo.st_birthtimespec, ProcSymlink::FOLLOW); //throw FileError //sourceInfo.st_birthtime; -> only seconds-precision //sourceInfo.st_mtime; -> #else - setFileTime(targetFile, sourceInfo.st_mtime, ProcSymlink::FOLLOW); //throw FileError + setWriteTimeNative(targetFile, sourceInfo.st_mtime, ProcSymlink::FOLLOW); //throw FileError #endif + InSyncAttributes newAttrib; + newAttrib.fileSize = sourceInfo.st_size; +#ifdef ZEN_MAC + newAttrib.modificationTime = sourceInfo.st_mtimespec.tv_sec; //use same time variable like setWriteTimeNative() for consistency +#else + newAttrib.modificationTime = sourceInfo.st_mtime; +#endif + newAttrib.sourceFileId = extractFileId(sourceInfo); + newAttrib.targetFileId = extractFileId(targetInfo); return newAttrib; } #endif @@ -2331,9 +2346,9 @@ copyFileWindowsDefault(::CopyFileEx) copyFileWindowsBackupStream(::BackupRead/: InSyncAttributes zen::copyNewFile(const Zstring& sourceFile, const Zstring& targetFile, bool copyFilePermissions, //throw FileError, ErrorTargetExisting, ErrorFileLocked - const std::function& onUpdateCopyStatus) + const std::function& notifyProgress) { - const InSyncAttributes attr = copyFileOsSpecific(sourceFile, targetFile, onUpdateCopyStatus); //throw FileError, ErrorTargetExisting, ErrorFileLocked + const InSyncAttributes attr = copyFileOsSpecific(sourceFile, targetFile, notifyProgress); //throw FileError, ErrorTargetExisting, ErrorFileLocked //at this point we know we created a new file, so it's fine to delete it for cleanup! ZEN_ON_SCOPE_FAIL(try { removeFile(targetFile); } diff --git a/zen/file_access.h b/zen/file_access.h index ec5bda66..026027e7 100644 --- a/zen/file_access.h +++ b/zen/file_access.h @@ -63,7 +63,7 @@ struct InSyncAttributes InSyncAttributes copyNewFile(const Zstring& sourceFile, const Zstring& targetFile, bool copyFilePermissions, //throw FileError, ErrorTargetExisting, ErrorFileLocked //accummulated delta != file size! consider ADS, sparse, compressed files - const std::function& onUpdateCopyStatus); //may be nullptr; throw X! + const std::function& notifyProgress); //may be nullptr; throw X! } #endif //FILE_ACCESS_H_8017341345614857 diff --git a/zen/file_io.cpp b/zen/file_io.cpp index b385ce33..b4351ee8 100644 --- a/zen/file_io.cpp +++ b/zen/file_io.cpp @@ -152,6 +152,7 @@ FileInput::FileInput(const Zstring& filepath) : //throw FileError, ErrorFileLock #ifdef ZEN_WIN //destructor call would lead to member double clean-up!!! ZEN_ON_SCOPE_FAIL(::CloseHandle(fileHandle)); + #elif defined ZEN_LINUX || defined ZEN_MAC ZEN_ON_SCOPE_FAIL(::close(fileHandle)); #endif @@ -178,45 +179,38 @@ FileInput::~FileInput() } -size_t FileInput::read(void* buffer, size_t bytesToRead) //throw FileError; returns actual number of bytes read +size_t FileInput::tryRead(void* buffer, size_t bytesToRead) //throw FileError; may return short, only 0 means EOF! { - size_t bytesReadTotal = 0; + if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); - while (bytesToRead > 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check! - { #ifdef ZEN_WIN - //test for end of file: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365690%28v=vs.85%29.aspx - DWORD bytesRead = 0; - if (!::ReadFile(fileHandle, //__in HANDLE hFile, - buffer, //__out LPVOID lpBuffer, - static_cast(bytesToRead), //__in DWORD nNumberOfBytesToRead, - &bytesRead, //__out_opt LPDWORD lpNumberOfBytesRead, - nullptr)) //__inout_opt LPOVERLAPPED lpOverlapped - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"ReadFile"); + //posix ::read() semantics: test for end of file: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365690%28v=vs.85%29.aspx + DWORD bytesRead = 0; + if (!::ReadFile(fileHandle, //__in HANDLE hFile, + buffer, //__out LPVOID lpBuffer, + static_cast(bytesToRead), //__in DWORD nNumberOfBytesToRead, + &bytesRead, //__out_opt LPDWORD lpNumberOfBytesRead, + nullptr)) //__inout_opt LPOVERLAPPED lpOverlapped + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"ReadFile"); #elif defined ZEN_LINUX || defined ZEN_MAC - ssize_t bytesRead = 0; - do - { - bytesRead = ::read(fileHandle, buffer, bytesToRead); - } - while (bytesRead < 0 && errno == EINTR); //Compare copy_reg() in copy.c: ftp://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz + ssize_t bytesRead = 0; + do + { + bytesRead = ::read(fileHandle, buffer, bytesToRead); + } + while (bytesRead < 0 && errno == EINTR); //Compare copy_reg() in copy.c: ftp://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz - if (bytesRead < 0) - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"read"); + if (bytesRead < 0) + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"read"); #endif - if (bytesRead == 0) //"zero indicates end of file" - return bytesReadTotal; + if (static_cast(bytesRead) > bytesToRead) //better safe than sorry + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"ReadFile: buffer overflow."); //user should never see this - if (static_cast(bytesRead) > bytesToRead) //better safe than sorry - throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getFilePath())), L"ReadFile: buffer overflow."); //user should never see this + //if ::read is interrupted (EINTR) right in the middle, it will return successfully with "bytesRead < bytesToRead" => loop! - //if ::read is interrupted (EINTR) right in the middle, it will return successfully with "bytesRead < bytesToRead" => loop! - buffer = static_cast(buffer) + bytesRead; //suppress warning about pointer arithmetics on void* - bytesToRead -= bytesRead; - bytesReadTotal += bytesRead; - } - return bytesReadTotal; + return bytesRead; //"zero indicates end of file" } //---------------------------------------------------------------------------------------------------- @@ -355,8 +349,11 @@ void FileOutput::close() //throw FileError } -void FileOutput::write(const void* buffer, size_t bytesToWrite) //throw FileError +size_t FileOutput::tryWrite(const void* buffer, size_t bytesToWrite) //throw FileError; may return short! CONTRACT: bytesToWrite > 0 { + if (bytesToWrite == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + #ifdef ZEN_WIN DWORD bytesWritten = 0; //this parameter is NOT optional: http://blogs.msdn.com/b/oldnewthing/archive/2013/04/04/10407417.aspx if (!::WriteFile(fileHandle, //__in HANDLE hFile, @@ -370,28 +367,24 @@ void FileOutput::write(const void* buffer, size_t bytesToWrite) //throw FileErro throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"WriteFile: incomplete write."); //user should never see this #elif defined ZEN_LINUX || defined ZEN_MAC - while (bytesToWrite > 0) + ssize_t bytesWritten = 0; + do { - ssize_t bytesWritten = 0; - do - { - bytesWritten = ::write(fileHandle, buffer, bytesToWrite); - } - while (bytesWritten < 0 && errno == EINTR); - - if (bytesWritten <= 0) - { - if (bytesWritten == 0) //comment in safe-read.c suggests to treat this as an error due to buggy drivers - errno = ENOSPC; + bytesWritten = ::write(fileHandle, buffer, bytesToWrite); + } + while (bytesWritten < 0 && errno == EINTR); - THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"write"); - } - if (bytesWritten > static_cast(bytesToWrite)) //better safe than sorry - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"write: buffer overflow."); //user should never see this + if (bytesWritten <= 0) + { + if (bytesWritten == 0) //comment in safe-read.c suggests to treat this as an error due to buggy drivers + errno = ENOSPC; - //if ::write() is interrupted (EINTR) right in the middle, it will return successfully with "bytesWritten < bytesToWrite"! - buffer = static_cast(buffer) + bytesWritten; //suppress warning about pointer arithmetics on void* - bytesToWrite -= bytesWritten; + THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"write"); } + if (bytesWritten > static_cast(bytesToWrite)) //better safe than sorry + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getFilePath())), L"write: buffer overflow."); //user should never see this + + //if ::write() is interrupted (EINTR) right in the middle, it will return successfully with "bytesWritten < bytesToWrite"! #endif + return bytesWritten; } diff --git a/zen/file_io.h b/zen/file_io.h index 5bcf4189..261829cd 100644 --- a/zen/file_io.h +++ b/zen/file_io.h @@ -8,6 +8,7 @@ #define FILE_IO_H_89578342758342572345 #include "file_error.h" +#include "serialize.h" #ifdef ZEN_WIN #include "win.h" //includes "windows.h" @@ -54,9 +55,12 @@ public: FileInput(FileHandle handle, const Zstring& filepath); //takes ownership! ~FileInput(); - size_t read(void* buffer, size_t bytesToRead); //throw FileError; returns "bytesToRead", unless end of file! + //Windows: better use 64kB ?? https://technet.microsoft.com/en-us/library/cc938632.aspx + //Linux: use st_blksize? + size_t getBlockSize() const { return 128 * 1024; } + size_t tryRead(void* buffer, size_t bytesToRead); //throw FileError; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0! + FileHandle getHandle() { return fileHandle; } - size_t optimalBlockSize() const { return 128 * 1024; } private: FileHandle fileHandle; @@ -75,17 +79,39 @@ public: FileOutput(const Zstring& filepath, AccessFlag access); //throw FileError, ErrorTargetExisting FileOutput(FileHandle handle, const Zstring& filepath); //takes ownership! ~FileOutput(); - void close(); //throw FileError -> optional, but good place to catch errors when closing stream! - - void write(const void* buffer, size_t bytesToWrite); //throw FileError - FileHandle getHandle() { return fileHandle; } - size_t optimalBlockSize() const { return 128 * 1024; } FileOutput(FileOutput&& tmp); + size_t getBlockSize() const { return 128 * 1024; } + size_t tryWrite(const void* buffer, size_t bytesToWrite); //throw FileError; may return short! CONTRACT: bytesToWrite > 0 + + void close(); //throw FileError -> optional, but good place to catch errors when closing stream! + FileHandle getHandle() { return fileHandle; } + private: FileHandle fileHandle; }; + + +//native stream I/O convenience functions: + +template inline +BinContainer loadBinContainer(const Zstring& filePath, //throw FileError + const std::function& notifyProgress) //optional +{ + FileInput streamIn(filePath); //throw FileError, ErrorFileLocked + return unbufferedLoad(streamIn, notifyProgress); //throw FileError +} + + +template inline +void saveBinContainer(const Zstring& filePath, const BinContainer& buffer, //throw FileError + const std::function& notifyProgress) //optional +{ + FileOutput fileOut(filePath, FileOutput::ACC_OVERWRITE); // + unbufferedSave(buffer, fileOut, notifyProgress); //throw FileError + fileOut.close(); // +} } #endif //FILE_IO_H_89578342758342572345 diff --git a/zen/file_traverser.cpp b/zen/file_traverser.cpp index 7fd6c596..aa39e508 100644 --- a/zen/file_traverser.cpp +++ b/zen/file_traverser.cpp @@ -102,9 +102,6 @@ void zen::traverseFolder(const Zstring& dirPath, entryp = malloc(len); */ const size_t nameMax = std::max(::pathconf(dirPath.c_str(), _PC_NAME_MAX), 10000); //::pathconf may return long(-1) std::vector buffer(offsetof(struct ::dirent, d_name) + nameMax + 1); -#ifdef ZEN_MAC - std::vector bufferUtfDecomposed; -#endif DIR* folder = ::opendir(dirPath.c_str()); //directory must NOT end with path separator, except "/" if (!folder) @@ -131,26 +128,21 @@ void zen::traverseFolder(const Zstring& dirPath, (itemNameRaw[1] == 0 || (itemNameRaw[1] == '.' && itemNameRaw[2] == 0))) continue; #ifdef ZEN_MAC - //some file system abstraction layers fail to properly return decomposed UTF8: http://developer.apple.com/library/mac/#qa/qa1173/_index.html - //so we need to do it ourselves; perf: ~600 ns per conversion - //note: it's not sufficient to apply this in z_impl::compareFilenamesNoCase: if UTF8 forms differ, FFS assumes a rename in case sensitivity and - // will try to propagate the rename => this won't work if target drive reports a particular UTF8 form only! - if (CFStringRef cfStr = osx::createCFString(itemNameRaw)) + //see native_traverser_impl.h: + Zstring itemName; + try { - ZEN_ON_SCOPE_EXIT(::CFRelease(cfStr)); - - CFIndex lenMax = ::CFStringGetMaximumSizeOfFileSystemRepresentation(cfStr); //"could be much larger than the actual space required" => don't store in Zstring - if (lenMax > 0) - { - bufferUtfDecomposed.resize(lenMax); - if (::CFStringGetFileSystemRepresentation(cfStr, &bufferUtfDecomposed[0], lenMax)) //get decomposed UTF form (verified!) despite ambiguous documentation - itemNameRaw = &bufferUtfDecomposed[0]; - } + itemName = osx::convertToPrecomposedUtf(itemNameRaw); //throw SysError } - //const char* sampleDecomposed = "\x6f\xcc\x81.txt"; - //const char* samplePrecomposed = "\xc3\xb3.txt"; -#endif + catch (const SysError& e) //failure is not an item-level error since wo don't have the proper decomposed name!!! + { + throw FileError(replaceCpy(_("Cannot enumerate directory %x."), L"%x", fmtPath(dirPath)), + L"Failed to generate precomposed file name: " + fmtPath(itemNameRaw) + L"\n" + e.toString()); //too obscure to warrant translation + } + const Zstring& itemPath = appendSeparator(dirPath) + itemName; +#else const Zstring& itemPath = appendSeparator(dirPath) + itemNameRaw; +#endif struct ::stat statData = {}; try diff --git a/zen/format_unit.cpp b/zen/format_unit.cpp index cb92f1e4..3dfe805b 100644 --- a/zen/format_unit.cpp +++ b/zen/format_unit.cpp @@ -403,8 +403,8 @@ Opt zen::utcToMtpVariantTime(std::int64_t utcTime) //returns empty on er return NoValue(); double localVarTime = 0; - if (!SystemTimeToVariantTime(&systemTimeLocal, //_In_ LPSYSTEMTIME lpSystemTime, - &localVarTime)) //_Out_ DOUBLE *pvtime + if (!::SystemTimeToVariantTime(&systemTimeLocal, //_In_ LPSYSTEMTIME lpSystemTime, + &localVarTime)) //_Out_ DOUBLE *pvtime return NoValue(); return localVarTime; diff --git a/zen/guid.h b/zen/guid.h index 7f247508..e8326ce6 100644 --- a/zen/guid.h +++ b/zen/guid.h @@ -8,15 +8,12 @@ #define GUID_H_80425780237502345 #include -#include #ifdef __GNUC__ //boost should clean this mess up #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif - #include - #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/zen/i18n.h b/zen/i18n.h index 3f10ec81..340473f9 100644 --- a/zen/i18n.h +++ b/zen/i18n.h @@ -102,7 +102,7 @@ inline const TranslationHandler*& getTranslationInstance() { //avoid static destruction order fiasco: there may be accesses to "getTranslator()" during process shutdown e.g. show message in debug_minidump.cpp! - //=> use POD instead of a std::unique_ptr<>!!! + //=> use POD instead of a std::unique_ptr<>!!! static const TranslationHandler* inst = nullptr; //external linkage even in header! return inst; } @@ -113,7 +113,7 @@ struct CleanUpTranslationHandler ~CleanUpTranslationHandler() { const TranslationHandler*& handler = getTranslationInstance(); - assert(!handler); //clean up at a better time rather than during static destruction! potential MT issues!? + assert(!handler); //clean up at a better time rather than during static destruction! potential MT issues!? delete handler; handler = nullptr; //getTranslator() may be called even after static objects of this translation unit are destroyed! } diff --git a/zen/long_path_prefix.h b/zen/long_path_prefix.h deleted file mode 100644 index ef8e5dc8..00000000 --- a/zen/long_path_prefix.h +++ /dev/null @@ -1,134 +0,0 @@ -// ************************************************************************** -// * This file is part of the FreeFileSync project. It is distributed under * -// * GNU General Public License: http://www.gnu.org/licenses/gpl-3.0 * -// * Copyright (C) Zenju (zenju AT gmx DOT de) - All Rights Reserved * -// ************************************************************************** - -#ifndef LONG_PATH_PREFIX_H_3984678473567247567 -#define LONG_PATH_PREFIX_H_3984678473567247567 - -#include "win.h" -#include "zstring.h" - - -namespace zen -{ -//handle filepaths longer-equal 260 (== MAX_PATH) characters by applying \\?\-prefix; see: http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx#maxpath -/* -1. path must be absolute -2. if path is smaller than MAX_PATH nothing is changed! caveat: FindFirstFile() "Prepending the string "\\?\" does not allow access to the root directory." -3. path may already contain \\?\-prefix -*/ -Zstring applyLongPathPrefix(const Zstring& path); //noexcept -Zstring applyLongPathPrefixCreateDir(const Zstring& path); //noexcept -> special rule for ::CreateDirectory()/::CreateDirectoryEx(): MAX_PATH - 12(=^ 8.3 filepath) is threshold - -Zstring removeLongPathPrefix(const Zstring& path); //noexcept - - -Zstring ntPathToWin32Path(const Zstring& path); //noexcept -/* -http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx#NT_Namespaces - -As used by GetModuleFileNameEx() and symlinks (FSCTL_GET_REPARSE_POINT): - E.g.: - \??\C:\folder -> C:\folder - \SystemRoot -> C:\Windows -*/ -} - - - - - - - - - - - - - -//################## implementation ################## - -//there are two flavors of long path prefix: one for UNC paths, one for regular paths -const wchar_t LONG_PATH_PREFIX [] = L"\\\\?\\"; //don't use Zstring as global constant: avoid static initialization order problem in global namespace! -const wchar_t LONG_PATH_PREFIX_UNC[] = L"\\\\?\\UNC"; // - -template inline -Zstring applyLongPathPrefixImpl(const Zstring& path) -{ - assert(!path.empty()); //nicely check almost all WinAPI accesses! - assert(!zen::isWhiteSpace(path[0])); - - //http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx#naming_conventions - /* - - special names like ".NUL" create all kinds of trouble (e.g. CreateDirectory() reports success, but does nothing) - unless prefix is supplied => accept as limitation - */ - if (path.length() >= maxPath || //maximum allowed path length without prefix is (MAX_PATH - 1) - endsWith(path, L' ') || //by default all Win32 APIs trim trailing spaces and period, unless long path prefix is supplied! - endsWith(path, L'.')) //note: adding long path prefix might screw up relative paths "." and ".."! - if (!startsWith(path, LONG_PATH_PREFIX)) - { - if (startsWith(path, L"\\\\")) //UNC-name, e.g. \\zenju-pc\Users - return LONG_PATH_PREFIX_UNC + afterFirst(path, L'\\', zen::IF_MISSING_RETURN_NONE); //convert to \\?\UNC\zenju-pc\Users - else - return LONG_PATH_PREFIX + path; //prepend \\?\ prefix - } - return path; //fallback -} - - -inline -Zstring zen::applyLongPathPrefix(const Zstring& path) -{ - return applyLongPathPrefixImpl(path); -} - - -inline -Zstring zen::applyLongPathPrefixCreateDir(const Zstring& path) //noexcept -{ - //special rule for ::CreateDirectoryEx(): MAX_PATH - 12(=^ 8.3 filepath) is threshold - return applyLongPathPrefixImpl (path); -} - - -inline -Zstring zen::removeLongPathPrefix(const Zstring& path) //noexcept -{ - if (zen::startsWith(path, LONG_PATH_PREFIX)) - { - if (zen::startsWith(path, LONG_PATH_PREFIX_UNC)) //UNC-name - return replaceCpy(path, LONG_PATH_PREFIX_UNC, Zstr("\\"), false); - else - return replaceCpy(path, LONG_PATH_PREFIX, Zstr(""), false); - } - return path; //fallback -} - - -inline -Zstring zen::ntPathToWin32Path(const Zstring& path) //noexcept -{ - if (startsWith(path, L"\\??\\")) - return Zstring(path.c_str() + 4, path.length() - 4); - - if (startsWith(path, L"\\SystemRoot\\")) - { - DWORD bufSize = ::GetEnvironmentVariable(L"SystemRoot", nullptr, 0); - if (bufSize > 0) - { - std::vector buf(bufSize); - const DWORD charsWritten = ::GetEnvironmentVariable(L"SystemRoot", //_In_opt_ LPCTSTR lpName, - &buf[0], //_Out_opt_ LPTSTR lpBuffer, - bufSize); //_In_ DWORD nSize - if (0 < charsWritten && charsWritten < bufSize) - return replaceCpy(path, L"\\SystemRoot\\", appendSeparator(Zstring(&buf[0], charsWritten)), false); - } - } - - return path; -} - -#endif //LONG_PATH_PREFIX_H_3984678473567247567 diff --git a/zen/scope_guard.h b/zen/scope_guard.h index 1345447e..67eb3053 100644 --- a/zen/scope_guard.h +++ b/zen/scope_guard.h @@ -20,7 +20,7 @@ inline int getUncaughtExceptionCount() { return std::uncaught_exceptions(); } #elif defined ZEN_LINUX || defined ZEN_MAC //std::uncaught_exceptions() currently unsupported on GCC and Clang => clean up ASAP #ifdef ZEN_LINUX - static_assert(__GNUC__ < 5 || (__GNUC__ == 5 && (__GNUC_MINOR__ < 2 || (__GNUC_MINOR__ == 2 && __GNUC_PATCHLEVEL__ <= 1))), "check std::uncaught_exceptions support"); + static_assert(__GNUC__ < 5 || (__GNUC__ == 5 && (__GNUC_MINOR__ < 3 || (__GNUC_MINOR__ == 3 && __GNUC_PATCHLEVEL__ <= 1))), "check std::uncaught_exceptions support"); #else static_assert(__clang_major__ < 7 || (__clang_major__ == 7 && __clang_minor__ <= 0), "check std::uncaught_exceptions support"); #endif @@ -76,7 +76,7 @@ public: if (!dismissed) { #ifdef _MSC_VER - #pragma warning(suppress: 4127) //"conditional expression is constant" +#pragma warning(suppress: 4127) //"conditional expression is constant" #endif if (runMode != ScopeGuardRunMode::ON_EXIT) { @@ -86,7 +86,7 @@ public: } #ifdef _MSC_VER - #pragma warning(suppress: 4127) //"conditional expression is constant" +#pragma warning(suppress: 4127) //"conditional expression is constant" #endif if (runMode == ScopeGuardRunMode::ON_SUCCESS) fun_(); //throw X diff --git a/zen/serialize.h b/zen/serialize.h index 77cf657e..d7b4cd22 100644 --- a/zen/serialize.h +++ b/zen/serialize.h @@ -10,7 +10,7 @@ #include #include #include "string_base.h" -#include "file_io.h" +//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h! namespace zen @@ -32,8 +32,6 @@ class ByteArray; //ref-counted byte stream + guaranteed per class ByteArray //essentially a std::vector with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite { public: - ByteArray() : buffer(std::make_shared>()) {} - typedef std::vector::value_type value_type; typedef std::vector::iterator iterator; typedef std::vector::const_iterator const_iterator; @@ -46,45 +44,78 @@ public: void resize(size_t len) { buffer->resize(len); } size_t size() const { return buffer->size(); } - bool empty() const { return buffer->empty(); } + bool empty() const { return buffer->empty(); } inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; } private: - std::shared_ptr> buffer; //always bound! + std::shared_ptr> buffer { std::make_shared>() }; //always bound! //perf: shared_ptr indirection irrelevant: less than 1% slower! }; -//---------------------------------------------------------------------- -//functions based on binary container abstraction -template void saveBinStream(const Zstring& filepath, const BinContainer& cont, const std::function& onUpdateStatus); //throw FileError -template BinContainer loadBinStream(const Zstring& filepath, const std::function& onUpdateStatus); //throw FileError +/* +--------------------------------- +|Unbuffered Input Stream Concept| +--------------------------------- +struct UnbufferedInputStream +{ + size_t getBlockSize(); + size_t tryRead(void* buffer, size_t bytesToRead); //may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 +}; + +---------------------------------- +|Unbuffered Output Stream Concept| +---------------------------------- +struct UnbufferedOutputStream +{ + size_t getBlockSize(); + size_t tryWrite(const void* buffer, size_t bytesToWrite); //may return short! CONTRACT: bytesToWrite > 0 +}; +*/ +//functions based on unbuffered stream abstraction + +template +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, UnbufferedOutputStream& streamOut, const std::function& notifyProgress); //throw X + +template +void unbufferedSave(const BinContainer& buffer, UnbufferedOutputStream& streamOut, const std::function& notifyProgress); //throw X +template +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, const std::function& notifyProgress); //throw X /* ------------------------------ -|Binary Input Stream Concept| ------------------------------ -struct BinInputStream +------------------------------- +|Buffered Input Stream Concept| +------------------------------- +struct BufferedInputStream { - size_t read(void* data, size_t len); //return "len" bytes unless end of stream! + size_t read(void* buffer, size_t bytesToRead); //return "len" bytes unless end of stream! throw ? }; ------------------------------- -|Binary Output Stream Concept| ------------------------------- -struct BinOutputStream +-------------------------------- +|Buffered Output Stream Concept| +-------------------------------- +struct BufferedOutputStream { - void write(const void* data, size_t len); + void write(const void* buffer, size_t bytesToWrite); //throw ? }; */ +//functions based on buffered stream abstraction +template void writeNumber (BufferedOutputStream& stream, const N& num); // +template void writeContainer(BufferedOutputStream& stream, const C& str); //throw () +template < class BufferedOutputStream> void writeArray (BufferedOutputStream& stream, const void* data, size_t len); // -//binary input/output stream reference implementations: +//---------------------------------------------------------------------- +class UnexpectedEndOfStreamError {}; +template N readNumber (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) +template C readContainer(BufferedInputStream& stream); // +template < class BufferedInputStream> void readArray (BufferedInputStream& stream, void* data, size_t len); // +//buffered input/output stream reference implementations: template struct MemoryStreamIn { - MemoryStreamIn(const BinContainer& cont) : buffer(cont), pos(0) {} //this better be cheap! + MemoryStreamIn(const BinContainer& cont) : buffer(cont) {} //this better be cheap! size_t read(void* data, size_t len) //return "len" bytes unless end of stream! { @@ -98,7 +129,7 @@ struct MemoryStreamIn private: const BinContainer buffer; - size_t pos; + size_t pos = 0; }; template @@ -118,21 +149,6 @@ private: BinContainer buffer; }; -//---------------------------------------------------------------------- -//functions based on binary stream abstraction -template -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, const std::function& onNotifyCopyStatus); //optional - -template void writeNumber (BinOutputStream& stream, const N& num); // -template void writeContainer(BinOutputStream& stream, const C& str); //throw () -template < class BinOutputStream> void writeArray (BinOutputStream& stream, const void* data, size_t len); // - -//---------------------------------------------------------------------- -class UnexpectedEndOfStreamError {}; -template N readNumber (BinInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) -template C readContainer(BinInputStream& stream); // -template < class BinInputStream> void readArray (BinInputStream& stream, void* data, size_t len); // - @@ -141,67 +157,124 @@ template < class BinInputStream> void readArray (BinInputStream& stre //-----------------------implementation------------------------------- -template inline -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, size_t blockSize, - const std::function& onNotifyCopyStatus) //optional +template inline +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, //throw X + UnbufferedOutputStream& streamOut, // + const std::function& notifyProgress) //optional { - assert(blockSize > 0); - std::vector buffer(blockSize); + size_t unevenBytes = 0; + auto reportBytesProcessed = [&](size_t bytesReadOrWritten) + { + if (notifyProgress) + { + const size_t bytesToReport = (unevenBytes + bytesReadOrWritten) / 2; + notifyProgress(bytesToReport); //throw X! + unevenBytes = (unevenBytes + bytesReadOrWritten) - bytesToReport * 2; //unsigned arithmetics! + } + }; + + const size_t blockSizeIn = streamIn .getBlockSize(); + const size_t blockSizeOut = streamOut.getBlockSize(); + if (blockSizeIn == 0 || blockSizeOut == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + std::vector buffer; for (;;) { - const size_t bytesRead = streamIn.read(&buffer[0], buffer.size()); - streamOut.write(&buffer[0], bytesRead); + buffer.resize(buffer.size() + blockSizeIn); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSizeIn), blockSizeIn); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSizeIn + bytesRead); //caveat: unsigned arithmetics + + reportBytesProcessed(bytesRead); //throw X! - if (onNotifyCopyStatus) - onNotifyCopyStatus(bytesRead); //throw X! + size_t bytesRemaining = buffer.size(); + while (bytesRemaining >= blockSizeOut) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), blockSizeOut); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + buffer.erase(buffer.begin(), buffer.end() - bytesRemaining); - if (bytesRead != buffer.size()) //end of file + if (bytesRead == 0) //end of file break; } + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesRemaining); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + + if (unevenBytes != 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); } -template inline -void saveBinStream(const Zstring& filepath, //throw FileError - const BinContainer& cont, - const std::function& onUpdateStatus) //optional +template inline +void unbufferedSave(const BinContainer& buffer, + UnbufferedOutputStream& streamOut, //throw X + const std::function& notifyProgress) //optional { - MemoryStreamIn streamIn(cont); - FileOutput streamOut(filepath, zen::FileOutput::ACC_OVERWRITE); //throw FileError, (ErrorTargetExisting) - if (onUpdateStatus) onUpdateStatus(0); //throw X! - copyStream(streamIn, streamOut, streamOut.optimalBlockSize(), onUpdateStatus); //throw FileError + const size_t blockSize = streamOut.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesToWrite = std::min(bytesRemaining, blockSize); + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + if (notifyProgress) notifyProgress(bytesWritten); //throw X! + } } -template inline -BinContainer loadBinStream(const Zstring& filepath, //throw FileError - const std::function& onUpdateStatus) //optional +template inline +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, //throw X + const std::function& notifyProgress) //optional { - FileInput streamIn(filepath); //throw FileError, ErrorFileLocked - if (onUpdateStatus) onUpdateStatus(0); //throw X! - MemoryStreamOut streamOut; - copyStream(streamIn, streamOut, streamIn.optimalBlockSize(), onUpdateStatus); //throw FileError - return streamOut.ref(); + const size_t blockSize = streamIn.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + BinContainer buffer; + for (;;) + { + buffer.resize(buffer.size() + blockSize); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics + + if (notifyProgress) notifyProgress(bytesRead); //throw X! + + if (bytesRead == 0) //end of file + return buffer; + } } -template inline -void writeArray(BinOutputStream& stream, const void* data, size_t len) +template inline +void writeArray(BufferedOutputStream& stream, const void* data, size_t len) { stream.write(data, len); } -template inline -void writeNumber(BinOutputStream& stream, const N& num) +template inline +void writeNumber(BufferedOutputStream& stream, const N& num) { static_assert(IsArithmetic::value || IsSameType::value, ""); writeArray(stream, &num, sizeof(N)); } -template inline -void writeContainer(BinOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! +template inline +void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! { const auto len = cont.size(); writeNumber(stream, static_cast(len)); @@ -210,8 +283,8 @@ void writeContainer(BinOutputStream& stream, const C& cont) //don't even conside } -template inline -void readArray(BinInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError +template inline +void readArray(BufferedInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError { const size_t bytesRead = stream.read(data, len); if (bytesRead < len) @@ -219,8 +292,8 @@ void readArray(BinInputStream& stream, void* data, size_t len) //throw Unexpecte } -template inline -N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template inline +N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { static_assert(IsArithmetic::value || IsSameType::value, ""); N num = 0; @@ -229,8 +302,8 @@ N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError } -template inline -C readContainer(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template inline +C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { C cont; auto strLength = readNumber(stream); diff --git a/zen/shell_execute.h b/zen/shell_execute.h index 9b8e00f9..e6dcf7f5 100644 --- a/zen/shell_execute.h +++ b/zen/shell_execute.h @@ -82,12 +82,14 @@ void shellExecute(const Zstring& command, ExecutionType type) //throw FileError trim(commandTmp, true, false); //CommandLineToArgvW() does not like leading spaces std::vector argv; - int argc = 0; - if (LPWSTR* tmp = ::CommandLineToArgvW(commandTmp.c_str(), &argc)) - { + { + int argc = 0; + LPWSTR* tmp = ::CommandLineToArgvW(commandTmp.c_str(), &argc); + if (!tmp) + THROW_LAST_FILE_ERROR(_("Incorrect command line:") + L"\n" + commandTmp.c_str(), L"CommandLineToArgvW"); ZEN_ON_SCOPE_EXIT(::LocalFree(tmp)); std::copy(tmp, tmp + argc, std::back_inserter(argv)); - } + } Zstring filepath; Zstring arguments; @@ -106,7 +108,7 @@ void shellExecute(const Zstring& command, ExecutionType type) //throw FileError }; if (!shellExecuteImpl(fillExecInfo, type)) - THROW_LAST_FILE_ERROR(_("Incorrect command line:") + L"\nFile: " + fmtPath(filepath) + L"\nArg: " + copyStringTo(arguments), L"ShellExecuteEx"); + THROW_LAST_FILE_ERROR(_("Incorrect command line:") + L"\nFile: " + fmtPath(filepath) + L"\nArg: " + arguments.c_str(), L"ShellExecuteEx"); #elif defined ZEN_LINUX || defined ZEN_MAC /* diff --git a/zen/string_base.h b/zen/string_base.h index 96d46fc4..365e359e 100644 --- a/zen/string_base.h +++ b/zen/string_base.h @@ -266,6 +266,7 @@ public: void resize(size_t newSize, Char fillChar = 0); void swap(Zbase& other); void push_back(Char val) { operator+=(val); } //STL access + void pop_back(); Zbase& operator=(const Zbase& source); Zbase& operator=(Zbase&& tmp) noexcept; @@ -684,6 +685,16 @@ Zbase& Zbase::operator+=(Char ch) { return append(&ch, 1); } + + +template class SP, class AP> inline +void Zbase::pop_back() +{ + const size_t len = length(); + assert(len > 0); + if (len > 0) + resize(len - 1); +} } #endif //STRING_BASE_H_083217454562342526 diff --git a/zen/string_tools.h b/zen/string_tools.h index fc9fe806..92ca1654 100644 --- a/zen/string_tools.h +++ b/zen/string_tools.h @@ -77,6 +77,7 @@ template T copyStringTo(S&& str); template <> inline bool isWhiteSpace(char ch) { + assert(ch != 0); //std C++ does not consider 0 as white space //caveat 1: std::isspace() takes an int, but expects an unsigned char //caveat 2: some parts of UTF-8 chars are erroneously seen as whitespace, e.g. the a0 from "\xec\x8b\xa0" (MSVC) return static_cast(ch) < 128 && @@ -84,7 +85,11 @@ bool isWhiteSpace(char ch) } template <> inline -bool isWhiteSpace(wchar_t ch) { return std::iswspace(ch) != 0; } +bool isWhiteSpace(wchar_t ch) +{ + assert(ch != 0); //std C++ does not consider 0 as white space + return std::iswspace(ch) != 0; +} template inline @@ -402,7 +407,7 @@ template inline S printNumber(const T& format, const Num& number) //format a single number using ::sprintf { static_assert(IsSameType::Type, typename GetCharType::Type>::value, ""); - using CharType = typename GetCharType::Type; + using CharType = typename GetCharType::Type; const int BUFFER_SIZE = 128; CharType buffer[BUFFER_SIZE]; //zero-initialize? @@ -426,7 +431,7 @@ enum NumberType template inline S numberTo(const Num& number, Int2Type) //default number to string conversion using streams: convenient, but SLOW, SLOW, SLOW!!!! (~ factor of 20) { - using CharType = typename GetCharType::Type; + using CharType = typename GetCharType::Type; std::basic_ostringstream ss; ss << number; @@ -455,7 +460,7 @@ template inline void formatNegativeInteger(Num n, OutputIterator& it) { assert(n < 0); - using CharType = typename std::iterator_traits::value_type; + using CharType = typename std::iterator_traits::value_type; do { const Num tmp = n / 10; @@ -471,7 +476,7 @@ template inline void formatPositiveInteger(Num n, OutputIterator& it) { assert(n >= 0); - using CharType = typename std::iterator_traits::value_type; + using CharType = typename std::iterator_traits::value_type; do { const Num tmp = n / 10; @@ -487,7 +492,7 @@ S numberTo(const Num& number, Int2Type) { using CharType = typename GetCharType::Type; CharType buffer[2 + sizeof(Num) * 241 / 100]; //zero-initialize? - //it's generally faster to use a buffer than to rely on String::operator+=() (in)efficiency + //it's generally faster to use a buffer than to rely on String::operator+=() (in)efficiency //required chars (+ sign char): 1 + ceil(ln_10(256^sizeof(n) / 2 + 1)) -> divide by 2 for signed half-range; second +1 since one half starts with 1! // <= 1 + ceil(ln_10(256^sizeof(n))) =~ 1 + ceil(sizeof(n) * 2.4082) <= 2 + floor(sizeof(n) * 2.41) @@ -506,7 +511,7 @@ S numberTo(const Num& number, Int2Type) template inline S numberTo(const Num& number, Int2Type) { - using CharType = typename GetCharType::Type; + using CharType = typename GetCharType::Type; CharType buffer[1 + sizeof(Num) * 241 / 100]; //zero-initialize? //required chars: ceil(ln_10(256^sizeof(n))) =~ ceil(sizeof(n) * 2.4082) <= 1 + floor(sizeof(n) * 2.41) @@ -522,7 +527,7 @@ S numberTo(const Num& number, Int2Type) template inline Num stringTo(const S& str, Int2Type) //default string to number conversion using streams: convenient, but SLOW { - using CharType = typename GetCharType::Type; + using CharType = typename GetCharType::Type; Num number = 0; std::basic_istringstream(copyStringTo>(str)) >> number; return number; @@ -541,8 +546,8 @@ Num stringTo(const S& str, Int2Type) template Num extractInteger(const S& str, bool& hasMinusSign) //very fast conversion to integers: slightly faster than std::atoi, but more importantly: generic { - using CharType = typename GetCharType::Type; - + using CharType = typename GetCharType::Type; + const CharType* first = strBegin(str); const CharType* last = first + strLength(str); @@ -610,10 +615,10 @@ template inline S numberTo(const Num& number) { using TypeTag = Int2Type< - IsSignedInt ::value ? impl::NUM_TYPE_SIGNED_INT : - IsUnsignedInt::value ? impl::NUM_TYPE_UNSIGNED_INT : - IsFloat ::value ? impl::NUM_TYPE_FLOATING_POINT : - impl::NUM_TYPE_OTHER>; + IsSignedInt ::value ? impl::NUM_TYPE_SIGNED_INT : + IsUnsignedInt::value ? impl::NUM_TYPE_UNSIGNED_INT : + IsFloat ::value ? impl::NUM_TYPE_FLOATING_POINT : + impl::NUM_TYPE_OTHER>; return impl::numberTo(number, TypeTag()); } @@ -623,10 +628,10 @@ template inline Num stringTo(const S& str) { using TypeTag = Int2Type< - IsSignedInt ::value ? impl::NUM_TYPE_SIGNED_INT : - IsUnsignedInt::value ? impl::NUM_TYPE_UNSIGNED_INT : - IsFloat ::value ? impl::NUM_TYPE_FLOATING_POINT : - impl::NUM_TYPE_OTHER>; + IsSignedInt ::value ? impl::NUM_TYPE_SIGNED_INT : + IsUnsignedInt::value ? impl::NUM_TYPE_UNSIGNED_INT : + IsFloat ::value ? impl::NUM_TYPE_FLOATING_POINT : + impl::NUM_TYPE_OTHER>; return impl::stringTo(str, TypeTag()); } diff --git a/zen/thread.h b/zen/thread.h index 700d42dc..f747f965 100644 --- a/zen/thread.h +++ b/zen/thread.h @@ -181,9 +181,9 @@ auto runAsync(Function&& fun) template inline bool wait_for_all_timed(InputIterator first, InputIterator last, const Duration& duration) { - const std::chrono::steady_clock::time_point endTime = std::chrono::steady_clock::now() + duration; + const std::chrono::steady_clock::time_point stopTime = std::chrono::steady_clock::now() + duration; for (; first != last; ++first) - if (first->wait_until(endTime) != std::future_status::ready) + if (first->wait_until(stopTime) != std::future_status::ready) return false; //time elapsed return true; } diff --git a/zen/win.h b/zen/win.h deleted file mode 100644 index 8c1a1c7b..00000000 --- a/zen/win.h +++ /dev/null @@ -1,33 +0,0 @@ -// ************************************************************************** -// * This file is part of the FreeFileSync project. It is distributed under * -// * GNU General Public License: http://www.gnu.org/licenses/gpl-3.0 * -// * Copyright (C) Zenju (zenju AT gmx DOT de) - All Rights Reserved * -// ************************************************************************** - -#ifndef WIN_H_8701570183560183247891346363457 -#define WIN_H_8701570183560183247891346363457 - -#ifndef _WINSOCKAPI_ //prevent inclusion of winsock.h in windows.h: obsoleted by and conflicting with winsock2.h - #define _WINSOCKAPI_ -#endif - -//------------------------------------------------------ -#ifdef __WXMSW__ //we have wxWidgets - #include //includes "windows.h" - //------------------------------------------------------ -#else - //#define WIN32_LEAN_AND_MEAN - - #ifndef NOMINMAX - #define NOMINMAX - #endif - - #ifndef STRICT - #define STRICT //improve type checking - #endif - - #include -#endif -//------------------------------------------------------ - -#endif //WIN_H_8701570183560183247891346363457 diff --git a/zen/xml_io.cpp b/zen/xml_io.cpp index 5b2cfdca..ae69ade4 100644 --- a/zen/xml_io.cpp +++ b/zen/xml_io.cpp @@ -7,35 +7,42 @@ #include "xml_io.h" #include "file_access.h" #include "file_io.h" -#include "serialize.h" using namespace zen; XmlDoc zen::loadXmlDocument(const Zstring& filepath) //throw FileError { - //can't simply use zen::loadBinStream() due to the short-circuit xml-validation below! + //can't simply use zen::unbufferedLoad) due to the short-circuit xml-validation below! - FileInput fileStreamIn(filepath); //throw FileError - MemoryStreamOut memStreamOut; - { - //quick test whether input is an XML: avoid loading large binary files up front! - const std::string xmlBegin = " buf(xmlBegin.size() + strLength(BYTE_ORDER_MARK_UTF8)); + FileInput fileIn(filepath); //throw FileError, ErrorFileLocked + const size_t blockSize = fileIn.getBlockSize(); + const std::string xmlPrefix = " CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics - if (!startsWith(memStreamOut.ref(), xmlBegin) && - !startsWith(memStreamOut.ref(), BYTE_ORDER_MARK_UTF8 + xmlBegin)) //allow BOM! - throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filepath))); + //quick test whether input is an XML: avoid loading large binary files up front! + if (!xmlPrefixChecked && buffer.size() >= xmlPrefix.size() + strLength(BYTE_ORDER_MARK_UTF8)) + { + xmlPrefixChecked = true; + if (!startsWith(buffer, xmlPrefix) && + !startsWith(buffer, BYTE_ORDER_MARK_UTF8 + xmlPrefix)) //allow BOM! + throw FileError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtPath(filepath))); + } + + if (bytesRead == 0) //end of file + break; } - copyStream(fileStreamIn, memStreamOut, fileStreamIn.optimalBlockSize(), nullptr); //throw FileError - try { - return parse(memStreamOut.ref()); //throw XmlParsingError + return parse(buffer); //throw XmlParsingError } catch (const XmlParsingError& e) { @@ -50,19 +57,18 @@ XmlDoc zen::loadXmlDocument(const Zstring& filepath) //throw FileError void zen::saveXmlDocument(const XmlDoc& doc, const Zstring& filepath) //throw FileError { - std::string stream = serialize(doc); //noexcept + const std::string stream = serialize(doc); //noexcept //only update xml file if there are real changes try { if (getFilesize(filepath) == stream.size()) //throw FileError - if (loadBinStream(filepath, nullptr) == stream) //throw FileError + if (loadBinContainer(filepath, nullptr) == stream) //throw FileError return; } catch (FileError&) {} - FileOutput outputFile(filepath, FileOutput::ACC_OVERWRITE); //throw FileError - outputFile.write(stream.c_str(), stream.length()); // + saveBinContainer(filepath, stream, nullptr); //throw FileError } diff --git a/zen/zstring.cpp b/zen/zstring.cpp index 59f15f19..6d249e70 100644 --- a/zen/zstring.cpp +++ b/zen/zstring.cpp @@ -8,8 +8,7 @@ #include #ifdef ZEN_WIN - #include "dll.h" - //#include "win_ver.h" + #include "win.h" #endif using namespace zen; @@ -37,80 +36,67 @@ time per call | function #ifdef ZEN_WIN -namespace -{ -//try to call "CompareStringOrdinal" for low-level string comparison: unfortunately available not before Windows Vista! -//by a factor ~3 faster than old string comparison using "LCMapString" -typedef int (WINAPI* CompareStringOrdinalFunc)(LPCWSTR lpString1, int cchCount1, - LPCWSTR lpString2, int cchCount2, BOOL bIgnoreCase); -const SysDllFun compareStringOrdinal = SysDllFun(L"kernel32.dll", "CompareStringOrdinal"); -//watch for dependencies in global namespace!!! -} - - int cmpStringNoCase(const wchar_t* lhs, size_t lhsLen, const wchar_t* rhs, size_t rhsLen) { assert(std::find(lhs, lhs + lhsLen, 0) == lhs + lhsLen); //don't expect embedded nulls! assert(std::find(rhs, rhs + rhsLen, 0) == rhs + rhsLen); // - if (compareStringOrdinal) //this additional test has no noticeable performance impact +#ifdef ZEN_WIN_VISTA_AND_LATER + //"CompareStringOrdinal" (available since Windows Vista) is by a factor ~3 faster than old string comparison using "LCMapString" + const int rv = ::CompareStringOrdinal(lhs, //__in LPCWSTR lpString1, + static_cast(lhsLen), //__in int cchCount1, + rhs, //__in LPCWSTR lpString2, + static_cast(rhsLen), //__in int cchCount2, + true); //__in BOOL bIgnoreCase + if (rv <= 0) + throw std::runtime_error("Error comparing strings (CompareStringOrdinal). " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + else + return rv - 2; //convert to C-style string compare result +#else + //do NOT use "CompareString"; this function is NOT meant for file name comparisons (even with LOCALE_INVARIANT and SORT_STRINGSORT): for example "weiß" == "weiss"!!! + //the only reliable way to compare filepaths (with XP) is to call "CharUpper" or "LCMapString": + + const auto minSize = std::min(lhsLen, rhsLen); + + if (minSize == 0) //LCMapString does not allow input sizes of 0! + return static_cast(lhsLen) - static_cast(rhsLen); + + auto copyToUpperCase = [minSize](const wchar_t* strIn, wchar_t* strOut) + { + //faster than CharUpperBuff + wmemcpy or CharUpper + wmemcpy and same speed like ::CompareString() + if (::LCMapString(LOCALE_INVARIANT, //__in LCID Locale, + LCMAP_UPPERCASE, //__in DWORD dwMapFlags, + strIn, //__in LPCTSTR lpSrcStr, + static_cast(minSize), //__in int cchSrc, + strOut, //__out LPTSTR lpDestStr, + static_cast(minSize)) == 0) //__in int cchDest + throw std::runtime_error("Error comparing strings (LCMapString). " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + }; + + auto eval = [&](wchar_t* bufL, wchar_t* bufR) + { + copyToUpperCase(lhs, bufL); + copyToUpperCase(rhs, bufR); + + const int rv = ::wcsncmp(bufL, bufR, minSize); + if (rv != 0) + return rv; + + return static_cast(lhsLen) - static_cast(rhsLen); + }; + + if (minSize <= MAX_PATH) //performance optimization: stack { - const int rv = compareStringOrdinal(lhs, //__in LPCWSTR lpString1, - static_cast(lhsLen), //__in int cchCount1, - rhs, //__in LPCWSTR lpString2, - static_cast(rhsLen), //__in int cchCount2, - true); //__in BOOL bIgnoreCase - if (rv <= 0) - throw std::runtime_error("Error comparing strings (CompareStringOrdinal). " + std::string(__FILE__) + ":" + numberTo(__LINE__)); - else - return rv - 2; //convert to C-style string compare result + wchar_t bufferL[MAX_PATH] = {}; + wchar_t bufferR[MAX_PATH] = {}; + return eval(bufferL, bufferR); } - else //fallback + else //use freestore { - //do NOT use "CompareString"; this function is NOT accurate (even with LOCALE_INVARIANT and SORT_STRINGSORT): for example "weiß" == "weiss"!!! - //the only reliable way to compare filepaths (with XP) is to call "CharUpper" or "LCMapString": - - const auto minSize = std::min(lhsLen, rhsLen); - - if (minSize == 0) //LCMapString does not allow input sizes of 0! - return static_cast(lhsLen) - static_cast(rhsLen); - - auto copyToUpperCase = [minSize](const wchar_t* strIn, wchar_t* strOut) - { - //faster than CharUpperBuff + wmemcpy or CharUpper + wmemcpy and same speed like ::CompareString() - if (::LCMapString(LOCALE_INVARIANT, //__in LCID Locale, - LCMAP_UPPERCASE, //__in DWORD dwMapFlags, - strIn, //__in LPCTSTR lpSrcStr, - static_cast(minSize), //__in int cchSrc, - strOut, //__out LPTSTR lpDestStr, - static_cast(minSize)) == 0) //__in int cchDest - throw std::runtime_error("Error comparing strings (LCMapString). " + std::string(__FILE__) + ":" + numberTo(__LINE__)); - }; - - auto eval = [&](wchar_t* bufL, wchar_t* bufR) - { - copyToUpperCase(lhs, bufL); - copyToUpperCase(rhs, bufR); - - const int rv = ::wcsncmp(bufL, bufR, minSize); - if (rv != 0) - return rv; - - return static_cast(lhsLen) - static_cast(rhsLen); - }; - - if (minSize <= MAX_PATH) //performance optimization: stack - { - wchar_t bufferL[MAX_PATH] = {}; - wchar_t bufferR[MAX_PATH] = {}; - return eval(bufferL, bufferR); - } - else //use freestore - { - std::vector buffer(2 * minSize); - return eval(&buffer[0], &buffer[minSize]); - } + std::vector buffer(2 * minSize); + return eval(&buffer[0], &buffer[minSize]); } +#endif } -- cgit