From 88a8b528e20013c0aa3cc6bcd9659b0b5ddd9170 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Fri, 18 Apr 2014 17:20:07 +0200 Subject: 5.4 --- zen/FindFilePlus/FindFilePlus.vcxproj | 8 +- zen/FindFilePlus/find_file_plus.cpp | 43 +++- zen/FindFilePlus/find_file_plus.h | 3 +- zen/debug_log.h | 2 +- zen/dir_watcher.cpp | 21 +- zen/dst_hack.cpp | 8 +- zen/file_error.h | 11 +- zen/file_handling.cpp | 348 +++++++++++++++----------- zen/file_handling.h | 4 +- zen/file_io.cpp | 29 ++- zen/file_io.h | 2 +- zen/file_traverser.cpp | 297 +++++++++++----------- zen/file_traverser.h | 23 +- zen/file_update_handle.h | 33 +-- zen/i18n.h | 17 +- zen/last_error.h | 4 +- zen/long_path_prefix.h | 2 +- zen/privilege.cpp | 2 +- zen/recycler.cpp | 2 +- zen/string_base.h | 37 ++- zen/symlink_target.h | 11 +- zen/thread.h | 8 +- zen/tick_count.h | 11 +- zen/utf.h | 457 ++++++++++++++++++++++++++++++++++ zen/utf8.h | 315 ----------------------- zen/win_ver.h | 5 +- zen/zstring.h | 14 +- 27 files changed, 955 insertions(+), 762 deletions(-) create mode 100644 zen/utf.h delete mode 100644 zen/utf8.h (limited to 'zen') diff --git a/zen/FindFilePlus/FindFilePlus.vcxproj b/zen/FindFilePlus/FindFilePlus.vcxproj index 2c4256a6..a50239ab 100644 --- a/zen/FindFilePlus/FindFilePlus.vcxproj +++ b/zen/FindFilePlus/FindFilePlus.vcxproj @@ -80,10 +80,10 @@ FindFilePlus_$(Platform) FindFilePlus_$(Platform) FindFilePlus_$(Platform) - D:\Data\WinDDK\inc\ddk;D:\Data\WinDDK\inc\api;D:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include - D:\Data\WinDDK\inc\ddk;D:\Data\WinDDK\inc\api;D:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include - D:\Data\WinDDK\inc\ddk;D:\Data\WinDDK\inc\api;D:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include - D:\Data\WinDDK\inc\ddk;D:\Data\WinDDK\inc\api;D:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include + C:\Data\WinDDK\inc\ddk;C:\Data\WinDDK\inc\api;C:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include + C:\Data\WinDDK\inc\ddk;C:\Data\WinDDK\inc\api;C:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include + C:\Data\WinDDK\inc\ddk;C:\Data\WinDDK\inc\api;C:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include + C:\Data\WinDDK\inc\ddk;C:\Data\WinDDK\inc\api;C:\Data\WinDDK\inc\crt;$(WindowsSdkDir)\include;$(VCInstallDir)include diff --git a/zen/FindFilePlus/find_file_plus.cpp b/zen/FindFilePlus/find_file_plus.cpp index ad385668..9fba5f1a 100644 --- a/zen/FindFilePlus/find_file_plus.cpp +++ b/zen/FindFilePlus/find_file_plus.cpp @@ -265,14 +265,49 @@ void FileSearcher::readDirImpl(FileInformation& output) //throw FileError false); //__in BOOLEAN restartScan if (!NT_SUCCESS(rv)) { - if (rv == STATUS_NO_SUCH_FILE) //harmonize ntQueryDirectoryFile() error handling! failure to find a file on first call returns STATUS_NO_SUCH_FILE, - rv = STATUS_NO_MORE_FILES; //while on subsequent accesses returns STATUS_NO_MORE_FILES - //note: not all directories contain "., .." entries! E.g. a drive's root directory or NetDrive + ftp.gnu.org\CRYPTO.README" - //-> addon: this is NOT a directory, it looks like one in NetDrive, but it's a file in Opera + /* + fallback to default directory query method, if FileIdBothDirectoryInformation is not properly implemented + this is required for NetDrive mounted Webdav, e.g. www.box.net and NT4, 2000 remote drives, et al. + + NT status code | Win32 error code + --------------------------------|------------------------ + STATUS_INVALID_LEVEL | ERROR_INVALID_LEVEL + STATUS_NOT_SUPPORTED | ERROR_NOT_SUPPORTED + STATUS_UNEXPECTED_NETWORK_ERROR | ERROR_UNEXP_NET_ERR -> traverse network drive hosted by Win98 + STATUS_INVALID_PARAMETER | ERROR_INVALID_PARAMETER + STATUS_INVALID_NETWORK_RESPONSE | ERROR_BAD_NET_RESP + STATUS_INVALID_INFO_CLASS | ERROR_INVALID_PARAMETER + STATUS_UNSUCCESSFUL | ERROR_GEN_FAILURE + STATUS_ACCESS_VIOLATION | ERROR_NOACCESS ->FileIdBothDirectoryInformation on XP accessing UDF + STATUS_NO_SUCH_FILE | ERROR_FILE_NOT_FOUND + + rv == STATUS_NO_SUCH_FILE: + failure to find a file on first call returns STATUS_NO_SUCH_FILE, while subsequent accesses return STATUS_NO_MORE_FILES + note: not all directories contain ".", ".." entries! E.g. a drive's root directory or NetDrive + ftp.gnu.org\CRYPTO.README" + -> addon: this is NOT a directory, it looks like one in NetDrive, but it's a file in Opera + STATUS_NO_SUCH_FILE is abused by some citrix shares instead of "STATUS_INVALID_PARAMETER" so we treat it as such! + => since the directory is "truly empty" a fallback won't hurt + */ + if (rv == STATUS_INVALID_LEVEL || + rv == STATUS_NOT_SUPPORTED || + rv == STATUS_UNEXPECTED_NETWORK_ERROR || + rv == STATUS_INVALID_PARAMETER || + rv == STATUS_INVALID_NETWORK_RESPONSE || + rv == STATUS_INVALID_INFO_CLASS || + rv == STATUS_UNSUCCESSFUL || + rv == STATUS_ACCESS_VIOLATION || + rv == STATUS_NO_SUCH_FILE) + rv = STATUS_NOT_SUPPORTED; throw NtFileError(rv); //throws STATUS_NO_MORE_FILES when finished } + //for (NTSTATUS i = 0xC0000000L; i != -1; ++i) + //{ + // if (rtlNtStatusToDosError(i) == 59) //ERROR_UNEXP_NET_ERR + // __asm int 3; + //} + if (status.Information == 0) //except for the first call to call ::NtQueryDirectoryFile(): throw NtFileError(STATUS_BUFFER_OVERFLOW); //if buffer size is too small, return value is STATUS_SUCCESS and Information == 0 -> we don't expect this! } diff --git a/zen/FindFilePlus/find_file_plus.h b/zen/FindFilePlus/find_file_plus.h index 7d03d98b..3799a1e1 100644 --- a/zen/FindFilePlus/find_file_plus.h +++ b/zen/FindFilePlus/find_file_plus.h @@ -50,8 +50,7 @@ FindHandle openDir(const wchar_t* dirname); //returns nullptr on error, call ::G DLL_FUNCTION_DECLARATION bool readDir(FindHandle hnd, FileInformation& output); //returns false on error or if there are no more files; ::GetLastError() returns ERROR_NO_MORE_FILES in this case /* -warning:; this may also return file system implementation dependent error codes like ERROR_NOT_SUPPORTED, ERROR_INVALID_LEVEL, -ect. if "FileIdBothDirectoryInformation" is not supported! We need a fallback: +warning: may also return with ERROR_NOT_SUPPORTED if "FileIdBothDirectoryInformation" is not supported! We need a fallback: sometimes it's *not* sufficient to use fallback for NtQueryDirectoryFile() alone, we need to reset "hDir", since it may be fucked up by some poor file system layer implementation: - Samba before v3.0.22 (Mar 30, 2006) seems to have a bug which sucessfully returns 128 elements via NtQueryDirectoryFile() and FileIdBothDirectoryInformation, diff --git a/zen/debug_log.h b/zen/debug_log.h index 43b2fea3..6be81e89 100644 --- a/zen/debug_log.h +++ b/zen/debug_log.h @@ -56,7 +56,7 @@ public: const std::string& message) { const std::string logEntry = "[" + formatTime(FORMAT_TIME) + "] " + afterLast(sourceFile, ZEN_FILE_NAME_SEPARATOR) + - ", line " + toString(sourceRow) + ": " + message + "\n"; + ", line " + numberTo(sourceRow) + ": " + message + "\n"; const size_t bytesWritten = ::fwrite(logEntry.c_str(), 1, logEntry.size(), handle); if (std::ferror(handle) != 0 || bytesWritten != logEntry.size()) diff --git a/zen/dir_watcher.cpp b/zen/dir_watcher.cpp index 19c56d5a..e54c1b5b 100644 --- a/zen/dir_watcher.cpp +++ b/zen/dir_watcher.cpp @@ -22,9 +22,9 @@ #include "file_traverser.h" #endif - using namespace zen; + #ifdef FFS_WIN namespace { @@ -128,16 +128,7 @@ public: dirnamePf(appendSeparator(directory)), hDir(INVALID_HANDLE_VALUE) { - //these two privileges are required by ::CreateFile FILE_FLAG_BACKUP_SEMANTICS according to - //http://msdn.microsoft.com/en-us/library/aa363858(v=vs.85).aspx - try - { - activatePrivilege(SE_BACKUP_NAME); //throw FileError - activatePrivilege(SE_RESTORE_NAME); // - } - catch (const FileError&) {} - - hDir = ::CreateFile(applyLongPathPrefix(dirnamePf.c_str()).c_str(), + hDir = ::CreateFile(applyLongPathPrefix(dirnamePf).c_str(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, @@ -158,7 +149,7 @@ public: ~ReadChangesAsync() { - if (hDir != INVALID_HANDLE_VALUE) + if (hDir != INVALID_HANDLE_VALUE) //valid hDir is NOT an invariant, see move constructor! ::CloseHandle(hDir); } @@ -201,7 +192,7 @@ public: zen::ScopeGuard guardAio = zen::makeGuard([&] { //http://msdn.microsoft.com/en-us/library/aa363789(v=vs.85).aspx - if (::CancelIo(hDir) == TRUE) //cancel all async I/O related to this handle and thread + if (::CancelIo(hDir) != FALSE) //cancel all async I/O related to this handle and thread { DWORD bytesWritten = 0; ::GetOverlappedResult(hDir, &overlapped, &bytesWritten, true); //wait until cancellation is complete @@ -364,7 +355,7 @@ public: const std::shared_ptr& otherMe) : otherMe_(otherMe), dirs_(dirs) {} virtual void onFile (const Zchar* shortName, const Zstring& fullName, const FileInfo& details) {} - virtual void onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) {} + virtual HandleLink onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) { return LINK_SKIP; } virtual std::shared_ptr onDir(const Zchar* shortName, const Zstring& fullName) { dirs_.push_back(fullName); @@ -394,7 +385,7 @@ DirWatcher::DirWatcher(const Zstring& directory) : //throw FileError std::shared_ptr traverser; traverser = std::make_shared(fullDirList, traverser); //throw FileError - zen::traverseFolder(dirname, false, *traverser); //don't traverse into symlinks (analog to windows build) + zen::traverseFolder(dirname, *traverser); //don't traverse into symlinks (analog to windows build) //init pimpl_->dirname = directory; diff --git a/zen/dst_hack.cpp b/zen/dst_hack.cpp index 9457b966..6e5c2230 100644 --- a/zen/dst_hack.cpp +++ b/zen/dst_hack.cpp @@ -2,7 +2,7 @@ #include #include "basic_math.h" #include "long_path_prefix.h" -#include "utf8.h" +#include "utf.h" #include "assert_static.h" #include "int64.h" #include "file_error.h" @@ -169,7 +169,7 @@ FILETIME utcToLocal(const FILETIME& utcTime) //throw std::runtime_error const std::wstring errorMessage = _("Conversion error:") + L" FILETIME -> local FILETIME: " + L"(" + L"High: " + numberTo(utcTime.dwHighDateTime) + L" " + L"Low: " + numberTo(utcTime.dwLowDateTime) + L") " + L"\n\n" + getLastErrorFormatted(); - throw std::runtime_error(wideToUtf8(errorMessage)); + throw std::runtime_error(utfCvrtTo(errorMessage)); } return localTime; } @@ -187,7 +187,7 @@ FILETIME localToUtc(const FILETIME& localTime) //throw std::runtime_error const std::wstring errorMessage = _("Conversion error:") + L" local FILETIME -> FILETIME: " + L"(" + L"High: " + numberTo(localTime.dwHighDateTime) + L" " + L"Low: " + numberTo(localTime.dwLowDateTime) + L") " + L"\n\n" + getLastErrorFormatted(); - throw std::runtime_error(wideToUtf8(errorMessage)); + throw std::runtime_error(utfCvrtTo(errorMessage)); } return utcTime; } @@ -288,7 +288,7 @@ std::bitset getUtcLocalShift() { const std::wstring errorMessage = _("Conversion error:") + L" Unexpected UTC <-> local time shift: " + L"(" + numberTo(timeShiftSec) + L") " + L"\n\n" + getLastErrorFormatted(); - throw std::runtime_error(wideToUtf8(errorMessage)); + throw std::runtime_error(utfCvrtTo(errorMessage)); } std::bitset output(absValue); diff --git a/zen/file_error.h b/zen/file_error.h index 4565d0b7..622d6a47 100644 --- a/zen/file_error.h +++ b/zen/file_error.h @@ -9,7 +9,7 @@ #include #include "zstring.h" -#include "utf8.h" +#include "utf.h" #include "last_error.h" //we'll need this later anyway! namespace zen @@ -39,7 +39,7 @@ DEFINE_NEW_FILE_ERROR(ErrorFileLocked); //allow implicit UTF8 conversion: since std::wstring models a GUI string, convenience is more important than performance inline -std::wstring operator+(const std::wstring& lhs, const Zstring& rhs) { return std::wstring(lhs) += utf8CvrtTo(rhs); } +std::wstring operator+(const std::wstring& lhs, const Zstring& rhs) { return std::wstring(lhs) += utfCvrtTo(rhs); } //we musn't put our overloads in namespace std, but namespace zen (+ using directive) is sufficient @@ -48,10 +48,9 @@ inline std::wstring fmtFileName(const Zstring& filename) { std::wstring output; - output.reserve(filename.size() + 2); - output += L'\"'; - output += utf8CvrtTo(filename); - output += L'\"'; + output += L'\''; + output += utfCvrtTo(filename); + output += L'\''; return output; } } diff --git a/zen/file_handling.cpp b/zen/file_handling.cpp index 864fd61f..f004e09c 100644 --- a/zen/file_handling.cpp +++ b/zen/file_handling.cpp @@ -14,7 +14,6 @@ #include "file_io.h" #include "assert_static.h" #include -//#include #include "file_id_def.h" #ifdef FFS_WIN @@ -73,15 +72,15 @@ bool zen::dirExists(const Zstring& dirname) } -bool zen::symlinkExists(const Zstring& objname) +bool zen::symlinkExists(const Zstring& linkname) { #ifdef FFS_WIN - const DWORD ret = ::GetFileAttributes(applyLongPathPrefix(objname).c_str()); + const DWORD ret = ::GetFileAttributes(applyLongPathPrefix(linkname).c_str()); return ret != INVALID_FILE_ATTRIBUTES && (ret & FILE_ATTRIBUTE_REPARSE_POINT) != 0; #elif defined FFS_LINUX struct stat fileInfo = {}; - return ::lstat(objname.c_str(), &fileInfo) == 0 && + return ::lstat(linkname.c_str(), &fileInfo) == 0 && S_ISLNK(fileInfo.st_mode); //symbolic link #endif } @@ -143,7 +142,7 @@ void getFileAttrib(const Zstring& filename, FileAttrib& attr, ProcSymlink procSl FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS, //needed to open a directory nullptr); if (hFile == INVALID_HANDLE_VALUE) throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtFileName(filename)) + L"\n\n" + getLastErrorFormatted()); @@ -453,7 +452,7 @@ Zstring getFilenameFmt(const Zstring& filename, Function fun) //throw(); returns const DWORD rv = fun(filenameFmt.c_str(), //__in LPCTSTR lpszShortPath, &buffer[0], //__out LPTSTR lpszLongPath, bufferSize); //__in DWORD cchBuffer - if (rv == 0 || rv >= buffer.size()) + if (rv == 0 || rv >= bufferSize) return Zstring(); return &buffer[0]; @@ -478,7 +477,7 @@ Zstring findUnused8Dot3Name(const Zstring& filename) //find a unique 8.3 short n return output; } - throw std::runtime_error(std::string("100000000 files, one for each number, exist in this directory? You're kidding...\n") + utf8CvrtTo(pathPrefix)); + throw std::runtime_error(std::string("100000000 files, one for each number, exist in this directory? You're kidding...\n") + utfCvrtTo(pathPrefix)); } @@ -575,14 +574,10 @@ public: virtual void deleteTargetFile(const Zstring& targetFile) { assert(!fileExists(targetFile)); } - virtual void updateCopyStatus(UInt64 totalBytesTransferred) + virtual void updateCopyStatus(Int64 bytesDelta) { if (moveCallback) - { - const Int64 delta = to(totalBytesTransferred) - bytesReported; - moveCallback->updateStatus(delta); - bytesReported += delta; - } + moveCallback->updateStatus(bytesDelta); } private: @@ -592,7 +587,6 @@ private: const Zstring sourceFile_; const Zstring targetFile_; CallbackMoveFile* moveCallback; //optional - Int64 bytesReported; }; @@ -649,12 +643,13 @@ public: files_.push_back(std::make_pair(shortName, fullName)); } - virtual void onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) + virtual HandleLink onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) { if (details.dirLink) dirs_.push_back(std::make_pair(shortName, fullName)); else files_.push_back(std::make_pair(shortName, fullName)); + return LINK_SKIP; } virtual std::shared_ptr onDir(const Zchar* shortName, const Zstring& fullName) @@ -727,7 +722,7 @@ void moveDirectoryImpl(const Zstring& sourceDir, const Zstring& targetDir, Callb //traverse source directory one level TraverseOneLevel traverseCallback(fileList, dirList); - traverseFolder(sourceDir, false, traverseCallback); //traverse one level, don't follow symlinks + traverseFolder(sourceDir, traverseCallback); //traverse one level const Zstring targetDirPf = appendSeparator(targetDir); @@ -780,12 +775,13 @@ public: { m_files.push_back(fullName); } - virtual void onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) + virtual HandleLink onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) { if (details.dirLink) m_dirs.push_back(fullName); else m_files.push_back(fullName); + return LINK_SKIP; } virtual std::shared_ptr onDir(const Zchar* shortName, const Zstring& fullName) { @@ -836,7 +832,7 @@ void zen::removeDirectory(const Zstring& directory, CallbackRemoveDir* callback) { //get all files and directories from current directory (WITHOUT subdirectories!) FilesDirsOnlyTraverser traverser(fileList, dirList); - traverseFolder(directory, false, traverser); //don't follow symlinks + traverseFolder(directory, traverser); //don't follow symlinks } //delete directories recursively @@ -896,7 +892,9 @@ void zen::setFileTime(const Zstring& filename, const Int64& modificationTime, Pr FileUpdateHandle targetHandle(filename, [=] { return ::CreateFile(applyLongPathPrefix(filename).c_str(), - GENERIC_READ | GENERIC_WRITE, //use both when writing over network, see comment in file_io.cpp + FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES, + //avoids mysterious "access denied" when using "GENERIC_READ | GENERIC_WRITE" on a read-only file, even after read-only was removed right before: + //https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, @@ -917,11 +915,68 @@ void zen::setFileTime(const Zstring& filename, const Int64& modificationTime, Pr auto isNullTime = [](const FILETIME & ft) { return ft.dwLowDateTime == 0 && ft.dwHighDateTime == 0; }; - if (!::SetFileTime(targetHandle.get(), - isNullTime(creationTime) ? nullptr : &creationTime, - nullptr, - &lastWriteTime)) - throw FileError(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtFileName(filename)) + L"\n\n" + getLastErrorFormatted()); + if (!::SetFileTime(targetHandle.get(), //__in HANDLE hFile, + !isNullTime(creationTime) ? &creationTime : nullptr, //__in_opt const FILETIME *lpCreationTime, + nullptr, //__in_opt const FILETIME *lpLastAccessTime, + &lastWriteTime)) //__in_opt const FILETIME *lpLastWriteTime + { + auto lastErr = ::GetLastError(); + + //function may fail if file is read-only: https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 + if (lastErr == ERROR_ACCESS_DENIED) + { + //dynamically load windows API function: available with Windows Vista and later + typedef BOOL (WINAPI* SetFileInformationByHandleFunc)(HANDLE hFile, FILE_INFO_BY_HANDLE_CLASS FileInformationClass, LPVOID lpFileInformation, DWORD dwBufferSize); + + const SysDllFun setFileInformationByHandle(L"kernel32.dll", "SetFileInformationByHandle"); + if (setFileInformationByHandle) //if not: let the original error propagate! + { + auto setFileInfo = [&](FILE_BASIC_INFO basicInfo) //throw FileError; no const& since SetFileInformationByHandle() requires non-const parameter! + { + if (!setFileInformationByHandle(targetHandle.get(), //__in HANDLE hFile, + FileBasicInfo, //__in FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + &basicInfo, //__in LPVOID lpFileInformation, + sizeof(basicInfo))) //__in DWORD dwBufferSize + throw FileError(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtFileName(filename)) + L"\n\n" + getLastErrorFormatted()); + }; + + auto toLargeInteger = [](const FILETIME& ft) -> LARGE_INTEGER + { + LARGE_INTEGER tmp = {}; + tmp.LowPart = ft.dwLowDateTime; + tmp.HighPart = ft.dwHighDateTime; + return tmp; + }; + //--------------------------------------------------------------------------- + + BY_HANDLE_FILE_INFORMATION fileInfo = {}; + if (::GetFileInformationByHandle(targetHandle.get(), &fileInfo)) + if (fileInfo.dwFileAttributes & FILE_ATTRIBUTE_READONLY) + { + FILE_BASIC_INFO basicInfo = {}; //undocumented: file times of "0" stand for "don't change" + basicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL; //[!] the bug in the ticket above requires we set this together with file times!!! + basicInfo.LastWriteTime = toLargeInteger(lastWriteTime); // + if (!isNullTime(creationTime)) + basicInfo.CreationTime = toLargeInteger(creationTime); + + setFileInfo(basicInfo); //throw FileError + + try //... to restore original file attributes + { + FILE_BASIC_INFO basicInfo2 = {}; + basicInfo2.FileAttributes = fileInfo.dwFileAttributes; + setFileInfo(basicInfo2); //throw FileError + } + catch (FileError&) {} + + lastErr = ERROR_SUCCESS; + } + } + + if (lastErr != ERROR_SUCCESS) + throw FileError(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtFileName(filename)) + L"\n\n" + getLastErrorFormatted(lastErr)); + } + } #ifndef NDEBUG //dst hack: verify data written if (dst::isFatDrive(filename) && !dirExists(filename)) //throw() @@ -1322,7 +1377,6 @@ void createDirectory_straight(const Zstring& directory, const Zstring& templateD ZEN_ON_SCOPE_EXIT(::CloseHandle(hDir)); USHORT cmpState = COMPRESSION_FORMAT_DEFAULT; - DWORD bytesReturned = 0; ::DeviceIoControl(hDir, //handle to file or directory FSCTL_SET_COMPRESSION, //dwIoControlCode @@ -1364,8 +1418,11 @@ void createDirectoryRecursively(const Zstring& directory, const Zstring& templat return; } #elif defined FFS_LINUX - if (dirExists(directory)) - return; + if (somethingExists(directory)) + { + if (dirExists(directory)) + return; + } #endif else //if "not somethingExists" we need to create the parent directory { @@ -1485,7 +1542,7 @@ source attr | tf normal | tf compressed | tf encrypted | handled by ============|================================================================== --- | --- -C- E-- copyFileWindowsDefault --S | --S -CS E-S copyFileWindowsSparse - -C- | --- (NOK) -C- E-- copyFileWindowsDefault + -C- | -C- -C- E-- copyFileWindowsDefault -CS | -CS -CS E-S copyFileWindowsSparse E-- | E-- E-- E-- copyFileWindowsDefault E-S | E-- (NOK) E-- (NOK) E-- (NOK) copyFileWindowsDefault -> may fail with ERROR_DISK_FULL!! @@ -1502,14 +1559,10 @@ Note: - if target parent folder is compressed or encrypted, both attributes are //due to issues on non-NTFS volumes, we should use the copy-as-sparse routine only if required and supported! -bool canCopyAsSparse(HANDLE hSource, const Zstring& targetFile) //throw () +bool canCopyAsSparse(DWORD fileAttrSource, const Zstring& targetFile) //throw () { - BY_HANDLE_FILE_INFORMATION fileInfoSource = {}; - if (!::GetFileInformationByHandle(hSource, &fileInfoSource)) - return false; - - const bool sourceIsEncrypted = (fileInfoSource.dwFileAttributes & FILE_ATTRIBUTE_ENCRYPTED) != 0; - const bool sourceIsSparse = (fileInfoSource.dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0; + const bool sourceIsEncrypted = (fileAttrSource & FILE_ATTRIBUTE_ENCRYPTED) != 0; + const bool sourceIsSparse = (fileAttrSource & FILE_ATTRIBUTE_SPARSE_FILE) != 0; if (sourceIsEncrypted || !sourceIsSparse) //BackupRead() silently fails reading encrypted files! return false; //small perf optimization: don't check "targetFile" if not needed @@ -1544,18 +1597,23 @@ bool canCopyAsSparse(HANDLE hSource, const Zstring& targetFile) //throw () bool canCopyAsSparse(const Zstring& sourceFile, const Zstring& targetFile) //throw () { - HANDLE hFileSource = ::CreateFile(applyLongPathPrefix(sourceFile).c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //all shared modes are required to read files that are open in other applications - nullptr, - OPEN_EXISTING, - 0, - nullptr); - if (hFileSource == INVALID_HANDLE_VALUE) + //follow symlinks! + HANDLE hSource = ::CreateFile(applyLongPathPrefix(sourceFile).c_str(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //all shared modes are required to read files that are open in other applications + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (hSource == INVALID_HANDLE_VALUE) + return false; + ZEN_ON_SCOPE_EXIT(::CloseHandle(hSource)); + + BY_HANDLE_FILE_INFORMATION fileInfoSource = {}; + if (!::GetFileInformationByHandle(hSource, &fileInfoSource)) return false; - ZEN_ON_SCOPE_EXIT(::CloseHandle(hFileSource)); - return canCopyAsSparse(hFileSource, targetFile); //throw () + return canCopyAsSparse(fileInfoSource.dwFileAttributes, targetFile); //throw () } @@ -1567,7 +1625,7 @@ void copyFileWindowsSparse(const Zstring& sourceFile, { assert(canCopyAsSparse(sourceFile, targetFile)); - //comment suggests "FILE_FLAG_BACKUP_SEMANTICS + SE_BACKUP_NAME" may be needed: http://msdn.microsoft.com/en-us/library/windows/desktop/aa362509(v=vs.85).aspx + //try to get backup read and write privileges: who knows, maybe this helps solve some obscure "access denied" errors try { activatePrivilege(SE_BACKUP_NAME); } catch (const FileError&) {} try { activatePrivilege(SE_RESTORE_NAME); } @@ -1576,7 +1634,7 @@ void copyFileWindowsSparse(const Zstring& sourceFile, //open sourceFile for reading HANDLE hFileSource = ::CreateFile(applyLongPathPrefix(sourceFile).c_str(), GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //all shared modes are required to read files that are open in other applications + FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, //FILE_FLAG_OVERLAPPED must not be used! FILE_FLAG_SEQUENTIAL_SCAN | FILE_FLAG_BACKUP_SEMANTICS, //FILE_FLAG_NO_BUFFERING should not be used! @@ -1605,7 +1663,8 @@ void copyFileWindowsSparse(const Zstring& sourceFile, throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtFileName(sourceFile)) + L"\n\n" + getLastErrorFormatted()); //---------------------------------------------------------------------- - const DWORD validAttribs = FILE_ATTRIBUTE_READONLY | + const DWORD validAttribs = FILE_ATTRIBUTE_NORMAL | //"This attribute is valid only if used alone." + FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_ARCHIVE | //those two are not set properly (not worse than ::CopyFileEx()) @@ -1615,7 +1674,7 @@ void copyFileWindowsSparse(const Zstring& sourceFile, //create targetFile and open it for writing HANDLE hFileTarget = ::CreateFile(applyLongPathPrefix(targetFile).c_str(), GENERIC_READ | GENERIC_WRITE, //read access required for FSCTL_SET_COMPRESSION - FILE_SHARE_READ | FILE_SHARE_DELETE, + FILE_SHARE_DELETE, //FILE_SHARE_DELETE is required to rename file while handle is open! nullptr, CREATE_NEW, FILE_FLAG_SEQUENTIAL_SCAN | FILE_FLAG_BACKUP_SEMANTICS | (fileInfoSource.dwFileAttributes & validAttribs), @@ -1652,73 +1711,44 @@ void copyFileWindowsSparse(const Zstring& sourceFile, newAttrib->targetFileId = extractFileID(fileInfoTarget); } - //---------------------------------------------------------------------- - DWORD fsFlagsTarget = 0; - { - const DWORD bufferSize = 10000; - std::vector buffer(bufferSize); - - //full pathName need not yet exist! - if (!::GetVolumePathName(targetFile.c_str(), //__in LPCTSTR lpszFileName, - &buffer[0], //__out LPTSTR lpszVolumePathName, - bufferSize)) //__in DWORD cchBufferLength - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtFileName(targetFile)) + L"\n\n" + getLastErrorFormatted()); - - //GetVolumeInformationByHandleW would be a better solution, but it is supported beginning with Vista only! - if (!::GetVolumeInformation(&buffer[0], //__in_opt LPCTSTR lpRootPathName - nullptr, //__out_opt LPTSTR lpVolumeNameBuffer, - 0, //__in DWORD nVolumeNameSize, - nullptr, //__out_opt LPDWORD lpVolumeSerialNumber, - nullptr, //__out_opt LPDWORD lpMaximumComponentLength, - &fsFlagsTarget, //__out_opt LPDWORD lpFileSystemFlags, - nullptr, //__out LPTSTR lpFileSystemNameBuffer, - 0)) //__in DWORD nFileSystemNameSize - throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtFileName(targetFile)) + L"\n\n" + getLastErrorFormatted()); - } - - //---------------------------------------------------------------------- - const bool sourceIsCompressed = (fileInfoSource.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) != 0; - const bool sourceIsSparse = (fileInfoSource.dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0; - - const bool targetSupportsCompressed = (fsFlagsTarget & FILE_FILE_COMPRESSION ) != 0; - const bool targetSupportsSparse = (fsFlagsTarget & FILE_SUPPORTS_SPARSE_FILES) != 0; - - //---------------------------------------------------------------------- - if (sourceIsCompressed && targetSupportsCompressed) + //#################### copy NTFS compressed attribute ######################### + const bool sourceIsCompressed = (fileInfoSource.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) != 0; + const bool targetIsCompressed = (fileInfoTarget.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) != 0; //already set by CreateFile if target parent folder is compressed! + if (sourceIsCompressed && !targetIsCompressed) { USHORT cmpState = COMPRESSION_FORMAT_DEFAULT; DWORD bytesReturned = 0; - if (!DeviceIoControl(hFileTarget, //handle to file or directory - FSCTL_SET_COMPRESSION, //dwIoControlCode - &cmpState, //input buffer - sizeof(cmpState), //size of input buffer - nullptr, //lpOutBuffer - 0, //OutBufferSize - &bytesReturned, //number of bytes returned - nullptr)) //OVERLAPPED structure - { - //-> if target folder is encrypted this call will legitimately fail with ERROR_INVALID_FUNCTION - //throw FileError - } - } + if (!::DeviceIoControl(hFileTarget, //handle to file or directory + FSCTL_SET_COMPRESSION, //dwIoControlCode + &cmpState, //input buffer + sizeof(cmpState), //size of input buffer + nullptr, //lpOutBuffer + 0, //OutBufferSize + &bytesReturned, //number of bytes returned + nullptr)) //OVERLAPPED structure + {} //may legitimately fail with ERROR_INVALID_FUNCTION if: + // - target folder is encrypted + // - target volume does not support compressed attribute -> unlikely in this context + } + //############################################################################# //although it seems the sparse attribute is set automatically by BackupWrite, we are required to do this manually: http://support.microsoft.com/kb/271398/en-us //Quote: It is the responsibility of the backup utility to apply file attributes to a file after it is restored by using BackupWrite. //The application should retrieve the attributes by using GetFileAttributes prior to creating a backup with BackupRead. //If a file originally had the sparse attribute (FILE_ATTRIBUTE_SPARSE_FILE), the backup utility must explicitly set the - //attribute on the restored file. The attribute can be set by using the DeviceIoControl function with the FSCTL_SET_SPARSE flag. + //attribute on the restored file. - if (sourceIsSparse && targetSupportsSparse) + //if (sourceIsSparse && targetSupportsSparse) -> no need to check, this is our precondition! { DWORD bytesReturned = 0; - if (!DeviceIoControl(hFileTarget, //handle to file - FSCTL_SET_SPARSE, //dwIoControlCode - nullptr, //input buffer - 0, //size of input buffer - nullptr, //lpOutBuffer - 0, //OutBufferSize - &bytesReturned, //number of bytes returned - nullptr)) //OVERLAPPED structure + if (!::DeviceIoControl(hFileTarget, //handle to file + FSCTL_SET_SPARSE, //dwIoControlCode + nullptr, //input buffer + 0, //size of input buffer + nullptr, //lpOutBuffer + 0, //OutBufferSize + &bytesReturned, //number of bytes returned + nullptr)) //OVERLAPPED structure throw FileError(replaceCpy(_("Cannot write file attributes of %x."), L"%x", fmtFileName(targetFile)) + L"\n\n" + zen::getLastErrorFormatted() + L" (NTFS sparse)"); } @@ -1737,10 +1767,9 @@ void copyFileWindowsSparse(const Zstring& sourceFile, if (contextRead ) ::BackupRead (0, nullptr, 0, nullptr, true, false, &contextRead); //lpContext must be passed [...] all other parameters are ignored. if (contextWrite) ::BackupWrite(0, nullptr, 0, nullptr, true, false, &contextWrite); ); - //stream-copy sourceFile to targetFile - UInt64 totalBytesTransferred; //may be larger than file size! context information + ADS! bool eof = false; + bool silentFailure = true; //try to detect failure reading encrypted files do { DWORD bytesRead = 0; @@ -1772,18 +1801,21 @@ void copyFileWindowsSparse(const Zstring& sourceFile, if (bytesWritten != bytesRead) throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(targetFile)) + L"\n\n" + L"(incomplete write)"); - totalBytesTransferred += bytesRead; + //total bytes transferred may be larger than file size! context information + ADS or smaller (sparse, compressed)! //invoke callback method to update progress indicators if (callback) - callback->updateCopyStatus(totalBytesTransferred); //throw X! + callback->updateCopyStatus(Int64(bytesRead)); //throw X! + + if (bytesRead > 0) + silentFailure = false; } while (!eof); //DST hack not required, since both source and target volumes cannot be FAT! //::BackupRead() silently fails reading encrypted files -> double check! - if (totalBytesTransferred == 0U && UInt64(fileInfoSource.nFileSizeLow, fileInfoSource.nFileSizeHigh) != 0U) + if (silentFailure && UInt64(fileInfoSource.nFileSizeLow, fileInfoSource.nFileSizeHigh) != 0U) //note: there is no guaranteed ordering relation beween bytes transferred and file size! Consider ADS (>) and compressed/sparse files (<)! throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(sourceFile)) + L"\n\n" + L"(unknown error)"); @@ -1835,33 +1867,32 @@ DEFINE_NEW_FILE_ERROR(ErrorShouldCopyAsSparse); class ErrorHandling { public: - ErrorHandling() : shouldCopyAsSparse(false) {} - - void reportUserException(CallbackCopyFile& userCallback, const UInt64& bytesTransferred) - { - exceptionInUserCallback.reset(new std::pair(&userCallback, bytesTransferred)); - } + ErrorHandling() : shouldCopyAsSparse(false), exceptionInUserCallback(nullptr) {} + //call context: copyCallbackInternal() void reportErrorShouldCopyAsSparse() { shouldCopyAsSparse = true; } + void reportUserException(CallbackCopyFile& userCallback) { exceptionInUserCallback = &userCallback; } + void reportError(const std::wstring& message) { errorMsg = message; } + //call context: copyFileWindowsDefault() void evaluateErrors() //throw X { if (shouldCopyAsSparse) throw ErrorShouldCopyAsSparse(L"sparse dummy value"); if (exceptionInUserCallback) - exceptionInUserCallback->first->updateCopyStatus(exceptionInUserCallback->second); //rethrow (hopefully!) + exceptionInUserCallback->updateCopyStatus(0); //rethrow (hopefully!) if (!errorMsg.empty()) throw FileError(errorMsg); } private: - bool shouldCopyAsSparse; - std::wstring errorMsg; //these two are exclusive! - std::unique_ptr> exceptionInUserCallback; // + bool shouldCopyAsSparse; // + std::wstring errorMsg; //these are exclusive! + CallbackCopyFile* exceptionInUserCallback; // }; @@ -1879,7 +1910,9 @@ struct CallbackData CallbackCopyFile* const userCallback; //optional! ErrorHandling errorHandler; - FileAttrib newAttrib; //modified by CopyFileEx at start + FileAttrib newAttrib; //modified by CopyFileEx() at beginning + + Int64 bytesReported; //used internally to calculate bytes transferred delta }; @@ -1903,6 +1936,8 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, file time handling: ::CopyFileEx() will copy file modification time (only) over from source file AFTER the last invokation of this callback => it is possible to adapt file creation time of target in here, but NOT file modification time! + CAVEAT: if ::CopyFileEx() fails to set modification time, it silently ignores this error and returns success!!! + see procmon log in: https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3514569&group_id=234430 alternate data stream handling: CopyFileEx() processes multiple streams one after another, stream 1 is the file data stream and always available! @@ -1915,12 +1950,6 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, if (dwCallbackReason == CALLBACK_STREAM_SWITCH && //called up-front for every file (even if 0-sized) dwStreamNumber == 1) //consider ADS! { - if (canCopyAsSparse(hSourceFile, cbd.targetFile_)) //throw () - { - cbd.errorHandler.reportErrorShouldCopyAsSparse(); //use another copy routine! - return PROGRESS_CANCEL; - } - //#################### return source file attributes ################################ BY_HANDLE_FILE_INFORMATION fileInfoSrc = {}; if (!::GetFileInformationByHandle(hSourceFile, &fileInfoSrc)) @@ -1941,14 +1970,40 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, cbd.newAttrib.sourceFileId = extractFileID(fileInfoSrc); cbd.newAttrib.targetFileId = extractFileID(fileInfoTrg); + //#################### switch to sparse file copy if req. ####################### + if (canCopyAsSparse(fileInfoSrc.dwFileAttributes, cbd.targetFile_)) //throw () + { + cbd.errorHandler.reportErrorShouldCopyAsSparse(); //use a different copy routine! + return PROGRESS_CANCEL; + } + //#################### copy file creation time ################################ ::SetFileTime(hDestinationFile, &fileInfoSrc.ftCreationTime, nullptr, nullptr); //no error handling! - //############################################################################## + + //#################### copy NTFS compressed attribute ######################### + const bool sourceIsCompressed = (fileInfoSrc.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) != 0; + const bool targetIsCompressed = (fileInfoTrg.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) != 0; //already set by CopyFileEx if target parent folder is compressed! + if (sourceIsCompressed && !targetIsCompressed) + { + USHORT cmpState = COMPRESSION_FORMAT_DEFAULT; + DWORD bytesReturned = 0; + if (!::DeviceIoControl(hDestinationFile, //handle to file or directory + FSCTL_SET_COMPRESSION, //dwIoControlCode + &cmpState, //input buffer + sizeof(cmpState), //size of input buffer + nullptr, //lpOutBuffer + 0, //OutBufferSize + &bytesReturned, //number of bytes returned + nullptr)) //OVERLAPPED structure + {} //may legitimately fail with ERROR_INVALID_FUNCTION if + // - if target folder is encrypted + // - target volume does not support compressed attribute + //############################################################################# + } } //called after copy operation is finished - note: for 0-sized files this callback is invoked just ONCE! //if (totalFileSize.QuadPart == totalBytesTransferred.QuadPart && dwStreamNumber == 1) {} - if (cbd.userCallback) { //some odd check for some possible(?) error condition @@ -1960,13 +2015,14 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, nullptr, 0); try { - cbd.userCallback->updateCopyStatus(UInt64(totalBytesTransferred.QuadPart)); //throw X! + cbd.userCallback->updateCopyStatus(totalBytesTransferred.QuadPart - cbd.bytesReported); //throw X! + cbd.bytesReported = totalBytesTransferred.QuadPart; } catch (...) { //#warning migrate to std::exception_ptr when available - cbd.errorHandler.reportUserException(*cbd.userCallback, UInt64(totalBytesTransferred.QuadPart)); + cbd.errorHandler.reportUserException(*cbd.userCallback); return PROGRESS_CANCEL; } } @@ -1975,7 +2031,7 @@ DWORD CALLBACK copyCallbackInternal(LARGE_INTEGER totalFileSize, const bool supportNonEncryptedDestination = winXpOrLater(); //encrypted destination is not supported with Windows 2000 -const bool supportUnbufferedCopy = vistaOrLater(); +//const bool supportUnbufferedCopy = vistaOrLater(); //caveat: function scope static initialization is not thread-safe in VS 2010! @@ -1984,20 +2040,23 @@ void copyFileWindowsDefault(const Zstring& sourceFile, CallbackCopyFile* callback, FileAttrib* newAttrib) //throw FileError, ErrorTargetPathMissing, ErrorTargetExisting, ErrorFileLocked, ErrorShouldCopyAsSparse { + //try to get backup read and write privileges: who knows, maybe this helps solve some obscure "access denied" errors + try { activatePrivilege(SE_BACKUP_NAME); } + catch (const FileError&) {} + try { activatePrivilege(SE_RESTORE_NAME); } + catch (const FileError&) {} + zen::ScopeGuard guardTarget = zen::makeGuard([&] { try { removeFile(targetFile); } catch (...) {} }); //transactional behavior: guard just before starting copy, we don't trust ::CopyFileEx(), do we? ;) DWORD copyFlags = COPY_FILE_FAIL_IF_EXISTS; -#ifndef COPY_FILE_ALLOW_DECRYPTED_DESTINATION - const DWORD COPY_FILE_ALLOW_DECRYPTED_DESTINATION = 0x00000008; -#endif - if (supportNonEncryptedDestination) copyFlags |= COPY_FILE_ALLOW_DECRYPTED_DESTINATION; //allow copying from encrypted to non-encrytped location - if (supportUnbufferedCopy) //see http://blogs.technet.com/b/askperf/archive/2007/05/08/slow-large-file-copy-issues.aspx - copyFlags |= COPY_FILE_NO_BUFFERING; //no perf difference at worst, huge improvement for large files (20% in test NTFS -> NTFS) + //if (supportUnbufferedCopy) //see http://blogs.technet.com/b/askperf/archive/2007/05/08/slow-large-file-copy-issues.aspx + // copyFlags |= COPY_FILE_NO_BUFFERING; //no perf difference at worst, huge improvement for large files (20% in test NTFS -> NTFS) + //It's a shame this flag causes file corruption! https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3529683&group_id=234430 CallbackData cbd(callback, sourceFile, targetFile); @@ -2007,7 +2066,7 @@ void copyFileWindowsDefault(const Zstring& sourceFile, copyCallbackInternal, //__in_opt LPPROGRESS_ROUTINE lpProgressRoutine, &cbd, //__in_opt LPVOID lpData, nullptr, //__in_opt LPBOOL pbCancel, - copyFlags) == TRUE; //__in DWORD dwCopyFlags + copyFlags) != FALSE; //__in DWORD dwCopyFlags cbd.errorHandler.evaluateErrors(); //throw ?, process errors in callback first! if (!success) @@ -2141,18 +2200,15 @@ void copyFileLinux(const Zstring& sourceFile, }(); //copy contents of sourceFile to targetFile - UInt64 totalBytesTransferred; do { const size_t bytesRead = fileIn.read(&buffer[0], buffer.size()); //throw FileError fileOut.write(&buffer[0], bytesRead); //throw FileError - totalBytesTransferred += bytesRead; - //invoke callback method to update progress indicators if (callback) - callback->updateCopyStatus(totalBytesTransferred); //throw X! + callback->updateCopyStatus(Int64(bytesRead)); //throw X! } while (!fileIn.eof()); } @@ -2194,11 +2250,11 @@ void copyFileLinux(const Zstring& sourceFile, #endif -Zstring createTempName(const Zstring& filename) +Zstring findUnusedTempName(const Zstring& filename) { Zstring output = filename + zen::TEMP_FILE_ENDING; - //ensure uniqueness + //ensure uniqueness (+ minor race condition) for (int i = 1; somethingExists(output); ++i) output = filename + Zchar('_') + numberTo(i) + zen::TEMP_FILE_ENDING; @@ -2255,7 +2311,7 @@ void zen::copyFile(const Zstring& sourceFile, //throw FileError, ErrorTargetPath { //determine non-used temp file name "first": //using optimistic strategy: assume everything goes well, but recover on error -> minimize file accesses - temporary = createTempName(targetFile); + temporary = findUnusedTempName(targetFile); //retry copyFileSelectOs(sourceFile, temporary, callback, sourceAttr); //throw FileError diff --git a/zen/file_handling.h b/zen/file_handling.h index e90ad544..8848fbc2 100644 --- a/zen/file_handling.h +++ b/zen/file_handling.h @@ -21,7 +21,7 @@ struct CallbackCopyFile; bool fileExists (const Zstring& filename); //throw() check whether file or symlink exists bool dirExists (const Zstring& dirname); //throw() check whether directory or symlink exists -bool symlinkExists (const Zstring& objname); //throw() check whether a symbolic link exists +bool symlinkExists (const Zstring& linkname); //throw() check whether a symbolic link exists bool somethingExists(const Zstring& objname); //throw() check whether any object with this name exists //check whether two folders are located on the same (logical) volume @@ -108,7 +108,7 @@ struct CallbackCopyFile //callback functionality //may throw: //Linux: unconditionally //Windows: first exception is swallowed, updateCopyStatus() is then called again where it should throw again and exception will propagate as expected - virtual void updateCopyStatus(UInt64 totalBytesTransferred) = 0; + virtual void updateCopyStatus(Int64 bytesDelta) = 0; //accummulated delta != file size! consider ADS, sparse, compressed files }; diff --git a/zen/file_io.cpp b/zen/file_io.cpp index 462364da..3b3c244d 100644 --- a/zen/file_io.cpp +++ b/zen/file_io.cpp @@ -26,7 +26,7 @@ FileInput::FileInput(const Zstring& filename) : //throw FileError, ErrorNotExis #ifdef FFS_WIN fileHandle = ::CreateFile(zen::applyLongPathPrefix(filename).c_str(), GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //all shared modes are required to read open files that are shared by other applications + FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, @@ -62,12 +62,11 @@ FileInput::FileInput(const Zstring& filename) : //throw FileError, ErrorNotExis #endif { const ErrorCode lastError = getLastError(); - std::wstring errorMessage = replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + getLastErrorFormatted(lastError) + L" (open)"; if (errorCodeForNotExisting(lastError)) - throw ErrorNotExisting(errorMessage); + throw ErrorNotExisting(replaceCpy(_("Cannot find file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + getLastErrorFormatted(lastError)); - throw FileError(errorMessage); + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + getLastErrorFormatted(lastError) + L" (open)"); } } @@ -93,17 +92,23 @@ size_t FileInput::read(void* buffer, size_t bytesToRead) //returns actual number nullptr)) //__inout_opt LPOVERLAPPED lpOverlapped #elif defined FFS_LINUX const size_t bytesRead = ::fread(buffer, 1, bytesToRead, fileHandle); - if (::ferror(fileHandle) != 0) + if (::ferror(fileHandle) != 0) //checks status of stream, not fread()! #endif throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + getLastErrorFormatted() + L" (read)"); #ifdef FFS_WIN if (bytesRead < bytesToRead) //verify only! + eofReached = true; + #elif defined FFS_LINUX if (::feof(fileHandle) != 0) -#endif eofReached = true; + if (bytesRead < bytesToRead) + if (!eofReached) //pathologic!? + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + L"Incomplete read!"); +#endif + if (bytesRead > bytesToRead) throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + L"buffer overflow"); @@ -111,12 +116,6 @@ size_t FileInput::read(void* buffer, size_t bytesToRead) //returns actual number } -bool FileInput::eof() //end of file reached -{ - return eofReached; -} - - FileOutput::FileOutput(FileHandle handle, const Zstring& filename) : fileHandle(handle), filename_(filename) {} @@ -131,7 +130,7 @@ FileOutput::FileOutput(const Zstring& filename, AccessFlag access) : //throw Fil to use GENERIC_READ | GENERIC_WRITE for dwDesiredAccess than to use GENERIC_WRITE alone. The resulting code is faster, because the redirector can use the cache manager and send fewer SMBs with more data. This combination also avoids an issue where writing to a file across a network can occasionally return ERROR_ACCESS_DENIED. */ - FILE_SHARE_READ | FILE_SHARE_DELETE, //note: FILE_SHARE_DELETE is required to rename file while handle is open! + FILE_SHARE_DELETE, //FILE_SHARE_DELETE is required to rename file while handle is open! nullptr, access == ACC_OVERWRITE ? CREATE_ALWAYS : CREATE_NEW, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, @@ -192,10 +191,10 @@ void FileOutput::write(const void* buffer, size_t bytesToWrite) //throw FileErro nullptr)) //__inout_opt LPOVERLAPPED lpOverlapped #elif defined FFS_LINUX const size_t bytesWritten = ::fwrite(buffer, 1, bytesToWrite, fileHandle); - if (::ferror(fileHandle) != 0) + if (::ferror(fileHandle) != 0) //checks status of stream, not fwrite()! #endif throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + getLastErrorFormatted() + L" (w)"); //w -> distinguish from fopen error message! if (bytesWritten != bytesToWrite) //must be fulfilled for synchronous writes! - throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + L"(incomplete write)"); + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(filename_)) + L"\n\n" + L"Incomplete write!"); } diff --git a/zen/file_io.h b/zen/file_io.h index eb797b7b..33074d7e 100644 --- a/zen/file_io.h +++ b/zen/file_io.h @@ -35,7 +35,7 @@ public: ~FileInput(); size_t read(void* buffer, size_t bytesToRead); //throw FileError; returns actual number of bytes read - bool eof(); //end of file reached + bool eof() { return eofReached; } //end of file reached private: FileInput(const FileInput&); diff --git a/zen/file_traverser.cpp b/zen/file_traverser.cpp index ea6aa289..3acb0edf 100644 --- a/zen/file_traverser.cpp +++ b/zen/file_traverser.cpp @@ -43,9 +43,9 @@ bool tryReportingError(Command cmd, zen::TraverseCallback& callback) //return "t { switch (callback.onError(e.toString())) { - case TraverseCallback::TRAV_ERROR_RETRY: + case TraverseCallback::ON_ERROR_RETRY: break; - case TraverseCallback::TRAV_ERROR_IGNORE: + case TraverseCallback::ON_ERROR_IGNORE: return false; //default: // assert(false); @@ -57,7 +57,7 @@ bool tryReportingError(Command cmd, zen::TraverseCallback& callback) //return "t #ifdef FFS_WIN inline -bool extractFileInfoFromSymlink(const Zstring& linkName, zen::TraverseCallback::FileInfo& output) +bool getTargetInfoFromSymlink(const Zstring& linkName, zen::TraverseCallback::FileInfo& output) { //open handle to target of symbolic link HANDLE hFile = ::CreateFile(zen::applyLongPathPrefix(linkName).c_str(), @@ -65,7 +65,7 @@ bool extractFileInfoFromSymlink(const Zstring& linkName, zen::TraverseCallback:: FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS, //needed to open a directory nullptr); if (hFile == INVALID_HANDLE_VALUE) return false; @@ -179,7 +179,7 @@ struct Win32Traverser { struct DirHandle { - DirHandle() : searchHandle(nullptr), firstRead(true) {} + DirHandle() : searchHandle(nullptr), firstRead(true), firstData() {} HANDLE searchHandle; bool firstRead; @@ -268,7 +268,7 @@ struct FilePlusTraverser { hnd.searchHandle = ::openDir(applyLongPathPrefix(directory).c_str()); if (hnd.searchHandle == nullptr) - throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted()); + throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted() + L" (+)"); } static void destroy(DirHandle hnd) { ::closeDir(hnd.searchHandle); } //throw() @@ -285,32 +285,15 @@ struct FilePlusTraverser /* fallback to default directory query method, if FileIdBothDirectoryInformation is not properly implemented this is required for NetDrive mounted Webdav, e.g. www.box.net and NT4, 2000 remote drives, et al. - - NT status code | Win32 error code - --------------------------------|-------------------------- - STATUS_INVALID_LEVEL | ERROR_INVALID_LEVEL - STATUS_NOT_SUPPORTED | ERROR_NOT_SUPPORTED - STATUS_INVALID_PARAMETER | ERROR_INVALID_PARAMETER - STATUS_INVALID_NETWORK_RESPONSE | ERROR_BAD_NET_RESP - STATUS_INVALID_INFO_CLASS | ERROR_INVALID_PARAMETER - STATUS_UNSUCCESSFUL | ERROR_GEN_FAILURE - STATUS_ACCESS_VIOLATION | ERROR_NOACCESS ->FileIdBothDirectoryInformation on XP accessing UDF */ - - if (lastError == ERROR_INVALID_LEVEL || - lastError == ERROR_NOT_SUPPORTED || - lastError == ERROR_INVALID_PARAMETER || - lastError == ERROR_BAD_NET_RESP || - lastError == ERROR_UNEXP_NET_ERR || //traverse network drive hosted by Win98 - lastError == ERROR_GEN_FAILURE || - lastError == ERROR_NOACCESS) + if (lastError == ERROR_NOT_SUPPORTED) { fb(); //fallback should apply to whole directory sub-tree! return false; } //else we have a problem... report it: - throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted()); + throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted(lastError) + L" (+)"); } return true; } @@ -346,9 +329,8 @@ struct FilePlusTraverser class DirTraverser { public: - DirTraverser(const Zstring& baseDirectory, bool followSymlinks, zen::TraverseCallback& sink, zen::DstHackCallback* dstCallback) : + DirTraverser(const Zstring& baseDirectory, TraverseCallback& sink, DstHackCallback* dstCallback) : isFatFileSystem(dst::isFatDrive(baseDirectory)), - followSymlinks_(followSymlinks), volumeSerial(retrieveVolumeSerial(baseDirectory)) //return 0 on error { try //traversing certain folders with restricted permissions requires this privilege! (but copying these files may still fail) @@ -377,96 +359,103 @@ private: tryReportingError([&] { if (level == 100) //notify endless recursion - throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + _("Endless loop.")); + throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + _("Detected endless directory recursion.")); }, sink); typename Trav::DirHandle searchHandle; - const bool openSuccess = tryReportingError([&] - { - typedef Trav Trav; //f u VS! - Trav::create(directory, searchHandle); //throw FileError - }, sink); - - if (!openSuccess) - return; //ignored error + if (!tryReportingError([&] + { + typedef Trav Trav; //f u VS! + Trav::create(directory, searchHandle); //throw FileError + }, sink)) + return; //ignored error ZEN_ON_SCOPE_EXIT(typedef Trav Trav; Trav::destroy(searchHandle)); - typename Trav::FindData fileInfo = {}; + typename Trav::FindData findData = {}; auto fallback = [&] { this->traverse(directory, sink, level); }; //help VS2010 a little by avoiding too deeply nested lambdas - while ([&]() -> bool - { - bool gotEntry = false; - - typedef Trav Trav1; //f u VS! - tryReportingError([&] + for (;;) { - typedef Trav1 Trav; //f u VS! - gotEntry = Trav::getEntry(searchHandle, directory, fileInfo, fallback); //throw FileError - }, sink); + bool gotEntry = false; + tryReportingError([&] { typedef Trav Trav; /*VS 2010 bug*/ gotEntry = Trav::getEntry(searchHandle, directory, findData, fallback); }, sink); //throw FileError + if (!gotEntry) //no more items or ignored error + return; - return gotEntry; - }()) - { //skip "." and ".." - const Zchar* const shortName = Trav::getShortName(fileInfo); + const Zchar* const shortName = Trav::getShortName(findData); if (shortName[0] == L'.' && - (shortName[1] == 0 || (shortName[1] == L'.' && shortName[2] == 0))) + (shortName[1] == 0 || (shortName[1] == L'.' && shortName[2] == 0))) continue; const Zstring& fullName = appendSeparator(directory) + shortName; - if (Trav::isSymlink(fileInfo) && !followSymlinks_) //evaluate symlink directly + if (Trav::isSymlink(findData)) //check first! { - TraverseCallback::SymlinkInfo details; + TraverseCallback::SymlinkInfo linkInfo; try { - details.targetPath = getSymlinkRawTargetString(fullName); //throw FileError + linkInfo.targetPath = getSymlinkRawTargetString(fullName); //throw FileError } -#ifdef NDEBUG //Release - catch (FileError&) {} -#else - catch (FileError& e) { sink.onError(e.toString()); } //show broken symlink / access errors in debug build! -#endif + catch (FileError&) { assert(false); } + linkInfo.lastWriteTimeRaw = Trav::getModTime (findData); + linkInfo.dirLink = Trav::isDirectory(findData); //directory symlinks have this flag on Windows + + switch (sink.onSymlink(shortName, fullName, linkInfo)) + { + case TraverseCallback::LINK_FOLLOW: + { + //try to resolve symlink (and report error on failure!!!) + TraverseCallback::FileInfo targetInfo; + const bool validLink = tryReportingError([&] + { + if (!getTargetInfoFromSymlink(fullName, targetInfo)) + throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtFileName(fullName)) + L"\n\n" + getLastErrorFormatted()); + }, sink); + + if (validLink) + { + if (Trav::isDirectory(findData)) + { + if (const std::shared_ptr& rv = sink.onDir(shortName, fullName)) + traverse(fullName, *rv, level + 1); + } + else //a file + sink.onFile(shortName, fullName, targetInfo); + } + else //report broken symlink as file! + sink.onFile(shortName, fullName, TraverseCallback::FileInfo()); + } + break; - details.lastWriteTimeRaw = Trav::getModTime (fileInfo); - details.dirLink = Trav::isDirectory(fileInfo); //directory symlinks have this flag on Windows - sink.onSymlink(shortName, fullName, details); + case TraverseCallback::LINK_SKIP: + break; + } } - else if (Trav::isDirectory(fileInfo)) //a directory... or symlink that needs to be followed (for directory symlinks this flag is set too!) + else if (Trav::isDirectory(findData)) { - if (const std::shared_ptr rv = sink.onDir(shortName, fullName)) + if (const std::shared_ptr& rv = sink.onDir(shortName, fullName)) traverse(fullName, *rv, level + 1); } - else //a file or symlink that is followed... + else //a file { - TraverseCallback::FileInfo details; + TraverseCallback::FileInfo fileInfo; + Trav::extractFileInfo(findData, volumeSerial, fileInfo); - if (Trav::isSymlink(fileInfo)) //dereference symlinks! - { - extractFileInfoFromSymlink(fullName, details); //try to... - //keep details initial if symlink is broken - } - else + //####################################### DST hack ########################################### + if (isFatFileSystem) { - Trav::extractFileInfo(fileInfo, volumeSerial, details); - - //####################################### DST hack ########################################### - if (isFatFileSystem) - { - const dst::RawTime rawTime(Trav::getCreateTimeRaw(fileInfo), Trav::getModTimeRaw(fileInfo)); + const dst::RawTime rawTime(Trav::getCreateTimeRaw(findData), Trav::getModTimeRaw(findData)); - if (dst::fatHasUtcEncoded(rawTime)) //throw std::runtime_error - details.lastWriteTimeRaw = toTimeT(dst::fatDecodeUtcTime(rawTime)); //return real UTC time; throw (std::runtime_error) - else - markForDstHack.push_back(std::make_pair(fullName, Trav::getModTimeRaw(fileInfo))); - } - //####################################### DST hack ########################################### + if (dst::fatHasUtcEncoded(rawTime)) //throw std::runtime_error + fileInfo.lastWriteTimeRaw = toTimeT(dst::fatDecodeUtcTime(rawTime)); //return real UTC time; throw (std::runtime_error) + else + markForDstHack.push_back(std::make_pair(fullName, Trav::getModTimeRaw(findData))); } + //####################################### DST hack ########################################### - sink.onFile(shortName, fullName, details); + sink.onFile(shortName, fullName, fileInfo); } } } @@ -491,11 +480,11 @@ private: FileUpdateHandle updateHandle(i->first, [=] { return ::CreateFile(zen::applyLongPathPrefix(i->first).c_str(), - GENERIC_READ | GENERIC_WRITE, //use both when writing over network, see comment in file_io.cpp + FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS, //needed to open a directory nullptr); }); if (updateHandle.get() == INVALID_HANDLE_VALUE) @@ -543,7 +532,6 @@ private: FilenameTimeList markForDstHack; //####################################### DST hack ########################################### - const bool followSymlinks_; const DWORD volumeSerial; //may be 0! }; @@ -552,8 +540,7 @@ private: class DirTraverser { public: - DirTraverser(const Zstring& baseDirectory, bool followSymlinks, zen::TraverseCallback& sink, zen::DstHackCallback* dstCallback) : - followSymlinks_(followSymlinks) + DirTraverser(const Zstring& baseDirectory, zen::TraverseCallback& sink, zen::DstHackCallback* dstCallback) { const Zstring directoryFormatted = //remove trailing slash baseDirectory.size() > 1 && endsWith(baseDirectory, FILE_NAME_SEPARATOR) ? //exception: allow '/' @@ -580,23 +567,21 @@ private: tryReportingError([&] { if (level == 100) //notify endless recursion - throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + _("Endless loop.")); + throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + _("Detected endless directory recursion.")); }, sink); DIR* dirObj = nullptr; - tryReportingError([&] - { - dirObj = ::opendir(directory.c_str()); //directory must NOT end with path separator, except "/" + if (!tryReportingError([&] + { + dirObj = ::opendir(directory.c_str()); //directory must NOT end with path separator, except "/" if (!dirObj) throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted()); - }, sink); - - if (!dirObj) - return; //ignored error + }, sink)) + return; //ignored error ZEN_ON_SCOPE_EXIT(::closedir(dirObj)); //never close nullptr handles! -> crash - while (true) + for (;;) { struct ::dirent* dirEntry = nullptr; tryReportingError([&] @@ -604,7 +589,7 @@ private: if (::readdir_r(dirObj, reinterpret_cast< ::dirent*>(&buffer[0]), &dirEntry) != 0) throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtFileName(directory)) + L"\n\n" + getLastErrorFormatted()); }, sink); - if (!dirEntry) //no more items or ignore error + if (!dirEntry) //no more items or ignored error return; @@ -616,80 +601,88 @@ private: const Zstring& fullName = appendSeparator(directory) + shortName; - struct ::stat fileInfo = {}; - bool haveData = false; - tryReportingError([&] - { - if (::lstat(fullName.c_str(), &fileInfo) != 0) //lstat() does not resolve symlinks + struct ::stat statData = {}; + if (!tryReportingError([&] + { + if (::lstat(fullName.c_str(), &statData) != 0) //lstat() does not resolve symlinks throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtFileName(fullName)) + L"\n\n" + getLastErrorFormatted()); - haveData = true; - }, sink); - if (!haveData) - continue; //ignore error: skip file + }, sink)) + continue; //ignore error: skip file - if (S_ISLNK(fileInfo.st_mode)) + if (S_ISLNK(statData.st_mode)) //on Linux there is no distinction between file and directory symlinks! { - if (followSymlinks_) //on Linux Symlinks need to be followed to evaluate whether they point to a file or directory - { - if (::stat(fullName.c_str(), &fileInfo) != 0) //stat() resolves symlinks - { - sink.onFile(shortName, fullName, TraverseCallback::FileInfo()); //report broken symlink as file! - continue; - } + struct ::stat statDataTrg = {}; + bool validLink = ::stat(fullName.c_str(), &statDataTrg) == 0; //if "LINK_SKIP", a broken link is no error! - fileInfo.st_dev = 0; //id from dereferenced symlink is problematic, since renaming will consider the link, not the target! - fileInfo.st_ino = 0; // - } - else //evaluate symlink directly + TraverseCallback::SymlinkInfo linkInfo; + try { - TraverseCallback::SymlinkInfo details; - try - { - details.targetPath = getSymlinkRawTargetString(fullName); //throw FileError - } - catch (FileError& e) - { -#ifndef NDEBUG //show broken symlink / access errors in debug build! - sink.onError(e.toString()); -#endif - } + linkInfo.targetPath = getSymlinkRawTargetString(fullName); //throw FileError + } + catch (FileError&) { assert(false); } + linkInfo.lastWriteTimeRaw = statData.st_mtime; //UTC time (ANSI C format); unit: 1 second + linkInfo.dirLink = validLink && S_ISDIR(statDataTrg.st_mode); + //S_ISDIR and S_ISLNK are mutually exclusive on Linux => explicitly need to follow link - details.lastWriteTimeRaw = fileInfo.st_mtime; //UTC time(ANSI C format); unit: 1 second - details.dirLink = ::stat(fullName.c_str(), &fileInfo) == 0 && S_ISDIR(fileInfo.st_mode); - //S_ISDIR and S_ISLNK are mutually exclusive on Linux => explicitly need to follow link - sink.onSymlink(shortName, fullName, details); - continue; + switch (sink.onSymlink(shortName, fullName, linkInfo)) + { + case TraverseCallback::LINK_FOLLOW: + //try to resolve symlink (and report error on failure!!!) + validLink = tryReportingError([&] + { + if (validLink) return; //no need to check twice + if (::stat(fullName.c_str(), &statDataTrg) != 0) + throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtFileName(fullName)) + L"\n\n" + getLastErrorFormatted()); + }, sink); + + if (validLink) + { + if (S_ISDIR(statDataTrg.st_mode)) //a directory + { + if (const std::shared_ptr& rv = sink.onDir(shortName, fullName)) + traverse(fullName, *rv, level + 1); + } + else //a file + { + TraverseCallback::FileInfo fileInfo; + fileInfo.fileSize = zen::UInt64(statDataTrg.st_size); + fileInfo.lastWriteTimeRaw = statDataTrg.st_mtime; //UTC time (time_t format); unit: 1 second + //fileInfo.id = extractFileID(statDataTrg); -> id from dereferenced symlink is problematic, since renaming will consider the link, not the target! + sink.onFile(shortName, fullName, fileInfo); + } + } + else //report broken symlink as file! + sink.onFile(shortName, fullName, TraverseCallback::FileInfo()); + break; + + case TraverseCallback::LINK_SKIP: + break; } } - - //fileInfo contains dereferenced data in any case from here on - - if (S_ISDIR(fileInfo.st_mode)) //a directory... cannot be a symlink on Linux in this case + else if (S_ISDIR(statData.st_mode)) //a directory { - const std::shared_ptr rv = sink.onDir(shortName, fullName); - if (rv) + if (const std::shared_ptr& rv = sink.onDir(shortName, fullName)) traverse(fullName, *rv, level + 1); } - else //a file... (or symlink; pathological!) + else //a file { - TraverseCallback::FileInfo details; - details.fileSize = zen::UInt64(fileInfo.st_size); - details.lastWriteTimeRaw = fileInfo.st_mtime; //UTC time (time_t format); unit: 1 second - details.id = extractFileID(fileInfo); + TraverseCallback::FileInfo fileInfo; + fileInfo.fileSize = zen::UInt64(statData.st_size); + fileInfo.lastWriteTimeRaw = statData.st_mtime; //UTC time (time_t format); unit: 1 second + fileInfo.id = extractFileID(statData); - sink.onFile(shortName, fullName, details); + sink.onFile(shortName, fullName, fileInfo); } } } std::vector buffer; - const bool followSymlinks_; }; #endif } -void zen::traverseFolder(const Zstring& directory, bool followSymlinks, TraverseCallback& sink, DstHackCallback* dstCallback) +void zen::traverseFolder(const Zstring& directory, TraverseCallback& sink, DstHackCallback* dstCallback) { - DirTraverser(directory, followSymlinks, sink, dstCallback); + DirTraverser(directory, sink, dstCallback); } diff --git a/zen/file_traverser.h b/zen/file_traverser.h index ab46621f..c29d987d 100644 --- a/zen/file_traverser.h +++ b/zen/file_traverser.h @@ -17,9 +17,8 @@ namespace zen { -class TraverseCallback +struct TraverseCallback { -public: virtual ~TraverseCallback() {} struct FileInfo @@ -32,21 +31,27 @@ public: struct SymlinkInfo { Int64 lastWriteTimeRaw; //number of seconds since Jan. 1st 1970 UTC - Zstring targetPath; //may be empty if something goes wrong + Zstring targetPath; //optional: empty if something goes wrong bool dirLink; //"true": point to dir; "false": point to file (or broken Link on Linux) }; + enum HandleLink + { + LINK_FOLLOW, //dereferences link, then calls "onDir()" or "onFile()" + LINK_SKIP + }; + enum HandleError { - TRAV_ERROR_RETRY, - TRAV_ERROR_IGNORE + ON_ERROR_RETRY, + ON_ERROR_IGNORE }; //overwrite these virtual methods - virtual std::shared_ptr //nullptr: ignore directory, non-nullptr: traverse into + virtual std::shared_ptr //nullptr: ignore directory, non-nullptr: traverse into using the (new) callback /**/ onDir (const Zchar* shortName, const Zstring& fullName) = 0; virtual void onFile (const Zchar* shortName, const Zstring& fullName, const FileInfo& details) = 0; - virtual void onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) = 0; + virtual HandleLink onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) = 0; virtual HandleError onError (const std::wstring& errorText) = 0; }; @@ -64,12 +69,8 @@ struct DstHackCallback; //DST hack not required on Linux //custom traverser with detail information about files //directory may end with PATH_SEPARATOR void traverseFolder(const Zstring& directory, //throw(); - bool followSymlinks, TraverseCallback& sink, DstHackCallback* dstCallback = nullptr); //apply DST hack if callback is supplied -//followSymlinks: -//"true": Symlinks dereferenced and reported via onFile() and onDir() => onSymlink not used! -//"false": Symlinks directly reported via onSymlink(), directory symlinks are not followed } #endif // FILETRAVERSER_H_INCLUDED diff --git a/zen/file_update_handle.h b/zen/file_update_handle.h index 716048fd..3df69f10 100644 --- a/zen/file_update_handle.h +++ b/zen/file_update_handle.h @@ -18,25 +18,28 @@ public: attr(INVALID_FILE_ATTRIBUTES) { hFile = cmd(); - if (hFile != INVALID_HANDLE_VALUE) - return; - - const DWORD lastError = ::GetLastError(); - if (lastError == ERROR_ACCESS_DENIED) //function fails if file is read-only + if (hFile == INVALID_HANDLE_VALUE) { - //zen::ScopeGuard guardErrorCode = zen::makeGuard([&] { ::SetLastError(lastError); }); //transactional behavior: ensure cleanup (e.g. network drop) -> cref [!] - - //read-only file attribute may cause trouble: temporarily reset it - const DWORD tmpAttr = ::GetFileAttributes(filenameFmt.c_str()); - if (tmpAttr != INVALID_FILE_ATTRIBUTES && (tmpAttr & FILE_ATTRIBUTE_READONLY)) + //try to recover + if (::GetLastError() == ERROR_ACCESS_DENIED) //function fails if file is read-only { - if (::SetFileAttributes(filenameFmt.c_str(), FILE_ATTRIBUTE_NORMAL)) + //read-only file attribute may cause trouble: temporarily reset it + const DWORD tmpAttr = ::GetFileAttributes(filenameFmt.c_str()); + if (tmpAttr != INVALID_FILE_ATTRIBUTES) { - //guardErrorCode.dismiss(); - attr = tmpAttr; //"create" guard on read-only attribute + if (tmpAttr & FILE_ATTRIBUTE_READONLY) + { + if (::SetFileAttributes(filenameFmt.c_str(), FILE_ATTRIBUTE_NORMAL)) + { + //guardErrorCode.dismiss(); + attr = tmpAttr; //"create" guard on read-only attribute - //now try again - hFile = cmd(); + //now try again + hFile = cmd(); + } + } + else + ::SetLastError(ERROR_ACCESS_DENIED); } } } diff --git a/zen/i18n.h b/zen/i18n.h index cea4340d..8c740b23 100644 --- a/zen/i18n.h +++ b/zen/i18n.h @@ -9,10 +9,8 @@ #include #include -#include //thousands separator -#include "utf8.h" // -//thin layer to enable localization - without platform/library dependencies! +//minimal layer enabling text translation - without platform/library dependencies! #ifndef WXINTL_NO_GETTEXT_MACRO #error WXINTL_NO_GETTEXT_MACRO must be defined to deactivate wxWidgets underscore macro #endif @@ -36,7 +34,6 @@ struct TranslationHandler void setTranslator(TranslationHandler* newHandler = nullptr); //takes ownership TranslationHandler* getTranslator(); -std::wstring getThousandsSeparator(); @@ -93,18 +90,6 @@ void setTranslator(TranslationHandler* newHandler) { implementation::globalHandl inline TranslationHandler* getTranslator() { return implementation::globalHandler().get(); } - - -inline -std::wstring getThousandsSeparator() //consistency with sprintf(): just use the same values the C-runtime uses!!! -{ - //::setlocale (LC_ALL, ""); -> implicitly called by wxLocale - const lconv* localInfo = ::localeconv(); //always bound according to doc - return utf8CvrtTo(localInfo->thousands_sep); - // why not working? - // THOUSANDS_SEPARATOR = std::use_facet >(std::locale("")).thousands_sep(); - // DECIMAL_POINT = std::use_facet >(std::locale("")).decimal_point(); -} } #endif //I18_N_HEADER_3843489325045 diff --git a/zen/last_error.h b/zen/last_error.h index 28982b63..76850f6a 100644 --- a/zen/last_error.h +++ b/zen/last_error.h @@ -8,7 +8,7 @@ #define SYSTEMFUNCTIONS_H_INCLUDED #include -#include "utf8.h" +#include "utf.h" #include "i18n.h" #ifdef FFS_WIN @@ -116,7 +116,7 @@ std::wstring getLastErrorFormatted(ErrorCode lastError) replace(output, L"%x", numberTo(lastError)); output += L" "; - output += utf8CvrtTo(::strerror(lastError)); + output += utfCvrtTo(::strerror(lastError)); errno = lastError; //restore errno return output; diff --git a/zen/long_path_prefix.h b/zen/long_path_prefix.h index db3207d3..ed6308dc 100644 --- a/zen/long_path_prefix.h +++ b/zen/long_path_prefix.h @@ -15,7 +15,7 @@ namespace zen //handle filenames longer-equal 260 (== MAX_PATH) characters by applying \\?\-prefix; see: http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx#maxpath /* 1. path must be absolute -2. if path is smaller than MAX_PATH nothing is changed! +2. if path is smaller than MAX_PATH nothing is changed! caveat: FindFirstFile() "Prepending the string "\\?\" does not allow access to the root directory." 3. path may already contain \\?\-prefix */ Zstring applyLongPathPrefix(const Zstring& path); //throw() diff --git a/zen/privilege.cpp b/zen/privilege.cpp index 69c820f3..9d8f12a0 100644 --- a/zen/privilege.cpp +++ b/zen/privilege.cpp @@ -36,7 +36,7 @@ bool privilegeIsActive(LPCTSTR privilege) //throw FileError &alreadyGranted)) //__out LPBOOL pfResult throw FileError(replaceCpy(_("Cannot set privilege %x."), L"%x", std::wstring(L"\"") + privilege + L"\"") + L"\n\n" + getLastErrorFormatted()); - return alreadyGranted == TRUE; + return alreadyGranted != FALSE; } diff --git a/zen/recycler.cpp b/zen/recycler.cpp index e53b5f6a..5e5e56d4 100644 --- a/zen/recycler.cpp +++ b/zen/recycler.cpp @@ -157,7 +157,7 @@ bool zen::moveToRecycleBin(const Zstring& filename) //throw FileError } throw FileError(shortMsg + L"\n\n" + L"Glib Error Code " + numberTo(error->code) + /* L", " + - g_quark_to_string(error->domain) + */ L": " + utf8CvrtTo(error->message)); + g_quark_to_string(error->domain) + */ L": " + utfCvrtTo(error->message)); } #endif return true; diff --git a/zen/string_base.h b/zen/string_base.h index 45f65ab8..16731089 100644 --- a/zen/string_base.h +++ b/zen/string_base.h @@ -285,11 +285,6 @@ template class SP, class AP> inline Zbase::Zbase(Zbase&& tmp) { //rawStr = this->clone(tmp.rawStr); NO! do not increment ref-count of a potentially unshared string! We'd lose optimization opportunity of reusing it! //instead create a dummy string and swap: - if (canWrite(tmp.rawStr, 0)) //perf: this check saves about 4% + if (this->canWrite(tmp.rawStr, 0)) //perf: this check saves about 4% { rawStr = this->create(0); //no perf issue! see comment in default constructor rawStr[0] = 0; @@ -446,17 +441,17 @@ Zbase& Zbase::replace(size_t pos1, size_t n1, const const size_t newLen = oldLen - n1 + n2; - if (canWrite(rawStr, newLen)) + if (this->canWrite(rawStr, newLen)) { if (n1 < n2) //move remainder right -> std::copy_backward { std::copy_backward(rawStr + pos1 + n1, rawStr + oldLen + 1, rawStr + newLen + 1); //include null-termination - setLength(rawStr, newLen); + this->setLength(rawStr, newLen); } else if (n1 > n2) //shift left -> std::copy { std::copy(rawStr + pos1 + n1, rawStr + oldLen + 1, rawStr + pos1 + n2); //include null-termination - setLength(rawStr, newLen); + this->setLength(rawStr, newLen); } std::copy(str.data(), str.data() + n2, rawStr + pos1); @@ -470,7 +465,7 @@ Zbase& Zbase::replace(size_t pos1, size_t n1, const std::copy(str.data(), str.data() + n2, newStr + pos1); std::copy(rawStr + pos1 + n1, rawStr + oldLen + 1, newStr + pos1 + n2); //include null-termination - destroy(rawStr); + this->destroy(rawStr); rawStr = newStr; } return *this; @@ -480,12 +475,12 @@ Zbase& Zbase::replace(size_t pos1, size_t n1, const template class SP, class AP> inline void Zbase::resize(size_t newSize, Char fillChar) { - if (canWrite(rawStr, newSize)) + if (this->canWrite(rawStr, newSize)) { if (length() < newSize) std::fill(rawStr + length(), rawStr + newSize, fillChar); rawStr[newSize] = 0; - setLength(rawStr, newSize); //keep after call to length() + this->setLength(rawStr, newSize); //keep after call to length() } else { @@ -500,7 +495,7 @@ void Zbase::resize(size_t newSize, Char fillChar) else std::copy(rawStr, rawStr + newSize, newStr); - destroy(rawStr); + this->destroy(rawStr); rawStr = newStr; } } @@ -593,10 +588,10 @@ void Zbase::clear() { if (!empty()) { - if (canWrite(rawStr, 0)) + if (this->canWrite(rawStr, 0)) { rawStr[0] = 0; //keep allocated memory - setLength(rawStr, 0); // + this->setLength(rawStr, 0); // } else *this = Zbase(); @@ -614,13 +609,13 @@ void Zbase::swap(Zbase& other) template class SP, class AP> inline void Zbase::reserve(size_t minCapacity) //make unshared and check capacity { - if (!canWrite(rawStr, minCapacity)) + if (!this->canWrite(rawStr, minCapacity)) { //allocate a new string - Char* newStr = create(length(), std::max(minCapacity, length())); //reserve() must NEVER shrink the string: logical const! + Char* newStr = this->create(length(), std::max(minCapacity, length())); //reserve() must NEVER shrink the string: logical const! std::copy(rawStr, rawStr + length() + 1, newStr); //include 0-termination - destroy(rawStr); + this->destroy(rawStr); rawStr = newStr; } } @@ -629,11 +624,11 @@ void Zbase::reserve(size_t minCapacity) //make unshared and check template class SP, class AP> inline Zbase& Zbase::assign(const Char* source, size_t len) { - if (canWrite(rawStr, len)) + if (this->canWrite(rawStr, len)) { std::copy(source, source + len, rawStr); rawStr[len] = 0; //include null-termination - setLength(rawStr, len); + this->setLength(rawStr, len); } else *this = Zbase(source, len); @@ -650,7 +645,7 @@ Zbase& Zbase::append(const Char* source, size_t len) std::copy(source, source + len, rawStr + thisLen); rawStr[thisLen + len] = 0; - setLength(rawStr, thisLen + len); + this->setLength(rawStr, thisLen + len); return *this; } diff --git a/zen/symlink_target.h b/zen/symlink_target.h index 06239b5a..dfd5ebfa 100644 --- a/zen/symlink_target.h +++ b/zen/symlink_target.h @@ -64,16 +64,15 @@ Zstring getSymlinkRawTargetString(const Zstring& linkPath) //throw FileError #ifdef FFS_WIN //FSCTL_GET_REPARSE_POINT: http://msdn.microsoft.com/en-us/library/aa364571(VS.85).aspx - try //reading certain symlinks/junctions requires admin rights! This shall not cause an error in user mode! - { - activatePrivilege(SE_BACKUP_NAME); //throw FileError - } - catch (FileError&) {} + //reading certain symlinks/junctions requires admin rights! + try + { activatePrivilege(SE_BACKUP_NAME); } //throw FileError + catch (FileError&) {} //This shall not cause an error in user mode! const HANDLE hLink = ::CreateFile(applyLongPathPrefix(linkPath).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - 0, + nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, nullptr); diff --git a/zen/thread.h b/zen/thread.h index ba9a46e2..3fb73e70 100644 --- a/zen/thread.h +++ b/zen/thread.h @@ -8,8 +8,8 @@ #define BOOST_THREAD_WRAP_H //temporary solution until C++11 thread becomes fully available -#include #include +#include "fixed_list.h" #ifdef __MINGW32__ #pragma GCC diagnostic push @@ -63,7 +63,7 @@ public: private: class AsyncResult; - std::vector workload; + FixedList workload; //note: we cannot use std::vector: compiler error on GCC 4.7, probably a boost screw-up std::shared_ptr result; }; @@ -180,10 +180,10 @@ template inline void RunUntilFirstHit::addJob(Fun f) //f must return a std::unique_ptr containing a value on success { auto result2 = result; //VC11: this is ridiculous!!! - workload.push_back(boost::thread([result2, f] + workload.emplace_back([result2, f] { result2->reportFinished(f()); - })); + }); } diff --git a/zen/tick_count.h b/zen/tick_count.h index 37c7cc59..30b0295d 100644 --- a/zen/tick_count.h +++ b/zen/tick_count.h @@ -56,17 +56,17 @@ public: #endif TickVal() : val_() {} - TickVal(const NativeVal& val) : val_(val) {} + explicit TickVal(const NativeVal& val) : val_(val) {} inline friend std::int64_t operator-(const TickVal& lhs, const TickVal& rhs) { #ifdef FFS_WIN - assert_static(IsSignedInt::value); + assert_static(IsSignedInt::value); return lhs.val_.QuadPart - rhs.val_.QuadPart; #elif defined FFS_LINUX - assert_static(IsSignedInt::value); - assert_static(IsSignedInt::value); + assert_static(IsSignedInt::value); + assert_static(IsSignedInt::value); return static_cast(lhs.val_.tv_sec - rhs.val_.tv_sec) * 1000000000.0 + lhs.val_.tv_nsec - rhs.val_.tv_nsec; #endif } @@ -85,6 +85,7 @@ std::int64_t ticksPerSec() //return 0 on error LARGE_INTEGER frequency = {}; if (!::QueryPerformanceFrequency(&frequency)) return 0; + assert_static(sizeof(std::int64_t) >= sizeof(frequency.QuadPart)); return frequency.QuadPart; #elif defined FFS_LINUX @@ -107,7 +108,7 @@ TickVal getTicks() //return 0 on error if (::clock_gettime(CLOCK_MONOTONIC_RAW, &now) != 0) //CLOCK_MONOTONIC measures time reliably across processors! return TickVal(); #endif - return now; + return TickVal(now); } } diff --git a/zen/utf.h b/zen/utf.h new file mode 100644 index 00000000..2ba6eaa3 --- /dev/null +++ b/zen/utf.h @@ -0,0 +1,457 @@ +// ************************************************************************** +// * This file is part of the zenXML project. It is distributed under the * +// * Boost Software License, Version 1.0. See accompanying file * +// * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt. * +// * Copyright (C) ZenJu (zhnmju123 AT gmx DOT de) - All Rights Reserved * +// ************************************************************************** + +#ifndef STRING_UTF8_HEADER_01832479146991573473545 +#define STRING_UTF8_HEADER_01832479146991573473545 + +#include +#include +#include "string_tools.h" //copyStringTo + +namespace zen +{ +//convert all(!) char- and wchar_t-based "string-like" objects applying a UTF8 conversions (but only if necessary!) +template +TargetString utfCvrtTo(const SourceString& str); + +const char BYTE_ORDER_MARK_UTF8[] = "\xEF\xBB\xBF"; + +//---- explicit conversion: wide <-> utf8 ---- +template +CharString wideToUtf8(const WideString& str); //example: std::string tmp = wideToUtf8(L"abc"); + +template +WideString utf8ToWide(const CharString& str); //std::wstring tmp = utf8ToWide("abc"); + +//access unicode characters in UTF-encoded string (char- or wchar_t-based) +template +size_t unicodeLength(const UtfString& str); //return number of code points for UTF-encoded string + +template +size_t findUnicodePos(const UtfString& str, size_t unicodePos); //return position of unicode char in UTF-encoded string + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +//----------------------- implementation ---------------------------------- +namespace implementation +{ +typedef std::uint_fast32_t CodePoint; //must be at least four bytes +typedef std::uint_fast16_t Char16; //we need an unsigned type +typedef unsigned char Char8; + +const CodePoint CODE_POINT_MAX = 0x10ffff; + +const CodePoint HIGH_SURROGATE = 0xd800; +const CodePoint HIGH_SURROGATE_MAX = 0xdbff; + +const CodePoint LOW_SURROGATE = 0xdc00; +const CodePoint LOW_SURROGATE_MAX = 0xdfff; + + +template inline +void codePointToUtf16(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char16 +{ + //http://en.wikipedia.org/wiki/UTF-16 + assert(cp < HIGH_SURROGATE || LOW_SURROGATE_MAX < cp); //code points [0xd800, 0xdfff] are not allowed for UTF-16 + assert(cp <= CODE_POINT_MAX); + + if (cp < 0x10000) + writeOutput(static_cast(cp)); + else + { + cp -= 0x10000; + writeOutput(static_cast((cp >> 10) + HIGH_SURROGATE)); + writeOutput(static_cast((cp & 0x3ff) + LOW_SURROGATE)); + } +} + + +inline +size_t getUtf16Len(Char16 ch) //ch must be first code unit! +{ + const CodePoint cp = ch; + + if (HIGH_SURROGATE <= cp && cp <= HIGH_SURROGATE_MAX) + return 2; + else + { + assert(cp < LOW_SURROGATE || LOW_SURROGATE_MAX < cp); //NO low surrogate expected + return 1; + } +} + + +template inline +void utf16ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint +{ + assert_static(sizeof(typename std::iterator_traits::value_type) == 2); + + for ( ; first != last; ++first) + { + CodePoint cp = static_cast(*first); + if (HIGH_SURROGATE <= cp && cp <= HIGH_SURROGATE_MAX) + { + if (++first == last) + { + assert(false); //low surrogate expected + return; + } + assert(LOW_SURROGATE <= static_cast(*first) && static_cast(*first) <= LOW_SURROGATE_MAX); //low surrogate expected + cp = ((cp - HIGH_SURROGATE) << 10) + static_cast(*first) - LOW_SURROGATE + 0x10000; + } + else + assert(cp < LOW_SURROGATE || LOW_SURROGATE_MAX < cp); //NO low surrogate expected + + writeOutput(cp); + } +} + + +template inline +void codePointToUtf8(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char8 +{ + //http://en.wikipedia.org/wiki/UTF-8 + + if (cp < 0x80) + writeOutput(static_cast(cp)); + else if (cp < 0x800) + { + writeOutput(static_cast((cp >> 6 ) | 0xc0)); + writeOutput(static_cast((cp & 0x3f) | 0x80)); + } + else if (cp < 0x10000) + { + writeOutput(static_cast((cp >> 12 ) | 0xe0)); + writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); + writeOutput(static_cast((cp & 0x3f ) | 0x80)); + } + else + { + assert(cp <= CODE_POINT_MAX); + writeOutput(static_cast((cp >> 18 ) | 0xf0)); + writeOutput(static_cast(((cp >> 12) & 0x3f) | 0x80)); + writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); + writeOutput(static_cast((cp & 0x3f ) | 0x80)); + } +} + + +inline +size_t getUtf8Len(unsigned char ch) //ch must be first code unit! +{ + if (ch < 0x80) + return 1; + if (ch >> 5 == 0x6) + return 2; + if (ch >> 4 == 0xe) + return 3; + if (ch >> 3 == 0x1e) + return 4; + + assert(false); //no valid begin of UTF8 encoding + return 1; +} + + +template inline +void utf8ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint +{ + assert_static(sizeof(typename std::iterator_traits::value_type) == 1); + + for ( ; first != last; ++first) + { + auto getChar = [&](Char8& ch) -> bool + { + if (++first == last) + { + assert(false); //low surrogate expected + return false; + } + ch = static_cast(*first); + assert(ch >> 6 == 0x2); + return true; + }; + + Char8 ch = static_cast(*first); + switch (getUtf8Len(ch)) + { + case 1: + writeOutput(ch); + break; + case 2: + { + CodePoint cp = (ch & 0x1f) << 6; + if (!getChar(ch)) return; + cp += ch & 0x3f; + writeOutput(cp); + } + break; + case 3: + { + CodePoint cp = (ch & 0xf) << 12; + if (!getChar(ch)) return; + cp += (ch & 0x3f) << 6; + if (!getChar(ch)) return; + cp += ch & 0x3f; + writeOutput(cp); + } + break; + case 4: + { + CodePoint cp = (ch & 0x7) << 18; + if (!getChar(ch)) return; + cp += (ch & 0x3f) << 12; + if (!getChar(ch)) return; + cp += (ch & 0x3f) << 6; + if (!getChar(ch)) return; + cp += ch & 0x3f; + writeOutput(cp); + } + break; + default: + assert(false); + } + } +} + + +template inline +size_t unicodeLength(const CharString& str, char) //utf8 +{ + typedef typename GetCharType::Type CharType; + + const CharType* strFirst = strBegin(str); + const CharType* const strLast = strFirst + strLength(str); + + size_t len = 0; + while (strFirst < strLast) //[!] + { + ++len; + strFirst += getUtf8Len(*strFirst); //[!] + } + return len; +} + + +template inline +size_t unicodeLengthWide(const WideString& str, Int2Type<2>) //windows: utf16-wchar_t +{ + typedef typename GetCharType::Type CharType; + + const CharType* strFirst = strBegin(str); + const CharType* const strLast = strFirst + strLength(str); + + size_t len = 0; + while (strFirst < strLast) //[!] + { + ++len; + strFirst += getUtf16Len(*strFirst); //[!] + } + return len; +} + + +template inline +size_t unicodeLengthWide(const WideString& str, Int2Type<4>) //other OS: utf32-wchar_t +{ + return strLength(str); +} + + +template inline +size_t unicodeLength(const WideString& str, wchar_t) +{ + return unicodeLengthWide(str, Int2Type()); +} +} + + +template inline +size_t unicodeLength(const UtfString& str) //return number of code points +{ + return implementation::unicodeLength(str, typename GetCharType::Type()); +} + + +namespace implementation +{ +template inline +size_t findUnicodePos(const CharString& str, size_t unicodePos, char) //utf8-char +{ + typedef typename GetCharType::Type CharType; + + const CharType* strFirst = strBegin(str); + const size_t strLen = strLength(str); + + size_t utfPos = 0; + while (unicodePos-- > 0) + { + utfPos += getUtf8Len(strFirst[utfPos]); + + if (utfPos >= strLen) + return strLen; + } + return utfPos; +} + + +template inline +size_t findUnicodePosWide(const WideString& str, size_t unicodePos, Int2Type<2>) //windows: utf16-wchar_t +{ + typedef typename GetCharType::Type CharType; + + const CharType* strFirst = strBegin(str); + const size_t strLen = strLength(str); + + size_t utfPos = 0; + while (unicodePos-- > 0) + { + utfPos += getUtf16Len(strFirst[utfPos]); + + if (utfPos >= strLen) + return strLen; + } + return utfPos; +} + + +template inline +size_t findUnicodePosWide(const WideString& str, size_t unicodePos, Int2Type<4>) //other OS: utf32-wchar_t +{ + return std::min(strLength(str), unicodePos); +} + + +template inline +size_t findUnicodePos(const UtfString& str, size_t unicodePos, wchar_t) +{ + return findUnicodePosWide(str, unicodePos, Int2Type()); +} +} + + +template inline +size_t findUnicodePos(const UtfString& str, size_t unicodePos) //return position of unicode char in UTF-encoded string +{ + return implementation::findUnicodePos(str, unicodePos, typename GetCharType::Type()); +} + +//------------------------------------------------------------------------------------------- + +namespace implementation +{ +template inline +WideString utf8ToWide(const CharString& str, Int2Type<2>) //windows: convert utf8 to utf16-wchar_t +{ + WideString output; + utf8ToCodePoint(strBegin(str), strBegin(str) + strLength(str), + [&](CodePoint cp) { codePointToUtf16(cp, [&](Char16 c) { output += static_cast(c); }); }); + return output; +} + + +template inline +WideString utf8ToWide(const CharString& str, Int2Type<4>) //other OS: convert utf8 to utf32-wchar_t +{ + WideString output; + utf8ToCodePoint(strBegin(str), strBegin(str) + strLength(str), + [&](CodePoint cp) { output += static_cast(cp); }); + return output; +} + + +template inline +CharString wideToUtf8(const WideString& str, Int2Type<2>) //windows: convert utf16-wchar_t to utf8 +{ + CharString output; + utf16ToCodePoint(strBegin(str), strBegin(str) + strLength(str), + [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); + return output; +} + + +template inline +CharString wideToUtf8(const WideString& str, Int2Type<4>) //other OS: convert utf32-wchar_t to utf8 +{ + CharString output; + std::for_each(strBegin(str), strBegin(str) + strLength(str), + [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); + return output; +} +} + + +template inline +WideString utf8ToWide(const CharString& str) +{ + assert_static((IsSameType::Type, char >::value)); + assert_static((IsSameType::Type, wchar_t>::value)); + + return implementation::utf8ToWide(str, Int2Type()); +} + + +template inline +CharString wideToUtf8(const WideString& str) +{ + assert_static((IsSameType::Type, char >::value)); + assert_static((IsSameType::Type, wchar_t>::value)); + + return implementation::wideToUtf8(str, Int2Type()); +} + +//------------------------------------------------------------------------------------------- + +template inline +TargetString utfCvrtTo(const SourceString& str, char, wchar_t) { return utf8ToWide(str); } + +template inline +TargetString utfCvrtTo(const SourceString& str, wchar_t, char) { return wideToUtf8(str); } + +template inline +TargetString utfCvrtTo(const SourceString& str, char, char) { return copyStringTo(str); } + +template inline +TargetString utfCvrtTo(const SourceString& str, wchar_t, wchar_t) { return copyStringTo(str); } + +template inline +TargetString utfCvrtTo(const SourceString& str) +{ + return utfCvrtTo(str, + typename GetCharType::Type(), + typename GetCharType::Type()); +} +} + +#endif //STRING_UTF8_HEADER_01832479146991573473545 diff --git a/zen/utf8.h b/zen/utf8.h deleted file mode 100644 index 242b729f..00000000 --- a/zen/utf8.h +++ /dev/null @@ -1,315 +0,0 @@ -// ************************************************************************** -// * This file is part of the zenXML project. It is distributed under the * -// * Boost Software License, Version 1.0. See accompanying file * -// * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt. * -// * Copyright (C) ZenJu (zhnmju123 AT gmx DOT de) - All Rights Reserved * -// ************************************************************************** - -#ifndef STRING_UTF8_HEADER_01832479146991573473545 -#define STRING_UTF8_HEADER_01832479146991573473545 - -#include -#include -#include "string_tools.h" //copyStringTo - -namespace zen -{ -//convert any(!) "string-like" object into target string by applying a UTF8 conversion (but only if necessary!) -template -TargetString utf8CvrtTo(const SourceString& str); - -//convert wide to utf8 string; example: std::string tmp = toUtf8(L"abc"); -template -CharString wideToUtf8(const WideString& str); - -//convert utf8 string to wide; example: std::wstring tmp = utf8To("abc"); -template -WideString utf8ToWide(const CharString& str); - -const char BYTE_ORDER_MARK_UTF8[] = "\xEF\xBB\xBF"; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//----------------------- implementation ---------------------------------- -namespace implementation -{ -typedef std::uint_fast32_t CodePoint; //must be at least four bytes -typedef std::uint_fast16_t Char16; //we need an unsigned type -typedef unsigned char Char8; - -const CodePoint CODE_POINT_MAX = 0x10ffff; - -const CodePoint HIGH_SURROGATE = 0xd800; -const CodePoint HIGH_SURROGATE_MAX = 0xdbff; - -const CodePoint LOW_SURROGATE = 0xdc00; -const CodePoint LOW_SURROGATE_MAX = 0xdfff; - - -template inline -void codePointToUtf16(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char16 -{ - //http://en.wikipedia.org/wiki/UTF-16 - assert(cp < HIGH_SURROGATE || LOW_SURROGATE_MAX < cp); //code points [0xd800, 0xdfff] are not allowed for UTF-16 - assert(cp <= CODE_POINT_MAX); - - if (cp < 0x10000) - writeOutput(static_cast(cp)); - else - { - cp -= 0x10000; - writeOutput(static_cast((cp >> 10) + HIGH_SURROGATE)); - writeOutput(static_cast((cp & 0x3ff) + LOW_SURROGATE)); - } -} - - -template inline -void utf16ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint -{ - assert_static(sizeof(typename std::iterator_traits::value_type) == 2); - - for ( ; first != last; ++first) - { - CodePoint cp = static_cast(*first); - if (HIGH_SURROGATE <= cp && cp <= HIGH_SURROGATE_MAX) - { - if (++first == last) - { - assert(false); //low surrogate expected - return; - } - assert(LOW_SURROGATE <= static_cast(*first) && static_cast(*first) <= LOW_SURROGATE_MAX); //low surrogate expected - cp = ((cp - HIGH_SURROGATE) << 10) + static_cast(*first) - LOW_SURROGATE + 0x10000; - } - else - assert(cp < LOW_SURROGATE || LOW_SURROGATE_MAX < cp); //NO low surrogate expected - - writeOutput(cp); - } -} - - -template inline -void codePointToUtf8(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char8 -{ - //http://en.wikipedia.org/wiki/UTF-8 - - if (cp < 0x80) - writeOutput(static_cast(cp)); - else if (cp < 0x800) - { - writeOutput(static_cast((cp >> 6 ) | 0xc0)); - writeOutput(static_cast((cp & 0x3f) | 0x80)); - } - else if (cp < 0x10000) - { - writeOutput(static_cast((cp >> 12 ) | 0xe0)); - writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); - writeOutput(static_cast((cp & 0x3f ) | 0x80)); - } - else - { - assert(cp <= CODE_POINT_MAX); - writeOutput(static_cast((cp >> 18 ) | 0xf0)); - writeOutput(static_cast(((cp >> 12) & 0x3f) | 0x80)); - writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); - writeOutput(static_cast((cp & 0x3f ) | 0x80)); - } -} - - -inline -size_t getUtf8Len(unsigned char ch) -{ - if (ch < 0x80) - return 1; - if (ch >> 5 == 0x6) - return 2; - if (ch >> 4 == 0xe) - return 3; - if (ch >> 3 == 0x1e) - return 4; - - assert(false); //no valid begin of UTF8 encoding - return 1; -} - - -template inline -void utf8ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint -{ - assert_static(sizeof(typename std::iterator_traits::value_type) == 1); - - for ( ; first != last; ++first) - { - auto getChar = [&](Char8& ch) -> bool - { - if (++first == last) - { - assert(false); //low surrogate expected - return false; - } - ch = static_cast(*first); - assert(ch >> 6 == 0x2); - return true; - }; - - Char8 ch = static_cast(*first); - switch (getUtf8Len(ch)) - { - case 1: - writeOutput(ch); - break; - case 2: - { - CodePoint cp = (ch & 0x1f) << 6; - if (!getChar(ch)) return; - cp += ch & 0x3f; - writeOutput(cp); - } - break; - case 3: - { - CodePoint cp = (ch & 0xf) << 12; - if (!getChar(ch)) return; - cp += (ch & 0x3f) << 6; - if (!getChar(ch)) return; - cp += ch & 0x3f; - writeOutput(cp); - } - break; - case 4: - { - CodePoint cp = (ch & 0x7) << 18; - if (!getChar(ch)) return; - cp += (ch & 0x3f) << 12; - if (!getChar(ch)) return; - cp += (ch & 0x3f) << 6; - if (!getChar(ch)) return; - cp += ch & 0x3f; - writeOutput(cp); - } - break; - default: - assert(false); - } - } -} - - -template inline -WideString utf8ToWide(const CharString& str, Int2Type<2>) //windows: convert utf8 to utf16 wchar_t -{ - WideString output; - utf8ToCodePoint(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf16(cp, [&](Char16 c) { output += static_cast(c); }); }); - return output; -} - - -template inline -WideString utf8ToWide(const CharString& str, Int2Type<4>) //other OS: convert utf8 to utf32 wchar_t -{ - WideString output; - utf8ToCodePoint(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { output += static_cast(cp); }); - return output; -} - - -template inline -CharString wideToUtf8(const WideString& str, Int2Type<2>) //windows: convert utf16-wchar_t to utf8 -{ - CharString output; - utf16ToCodePoint(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); - return output; -} - - -template inline -CharString wideToUtf8(const WideString& str, Int2Type<4>) //other OS: convert utf32-wchar_t to utf8 -{ - CharString output; - std::for_each(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); - return output; -} -} - - -template inline -WideString utf8ToWide(const CharString& str) -{ - assert_static((IsSameType::Type, char >::value)); - assert_static((IsSameType::Type, wchar_t>::value)); - - return implementation::utf8ToWide(str, Int2Type()); -} - - -template inline -CharString wideToUtf8(const WideString& str) -{ - assert_static((IsSameType::Type, char >::value)); - assert_static((IsSameType::Type, wchar_t>::value)); - - return implementation::wideToUtf8(str, Int2Type()); -} - - -//------------------------------------------------------------------------------------------- -template inline -TargetString utf8CvrtTo(const SourceString& str, char, wchar_t) { return utf8ToWide(str); } - -template inline -TargetString utf8CvrtTo(const SourceString& str, wchar_t, char) { return wideToUtf8(str); } - -template inline -TargetString utf8CvrtTo(const SourceString& str, char, char) { return copyStringTo(str); } - -template inline -TargetString utf8CvrtTo(const SourceString& str, wchar_t, wchar_t) { return copyStringTo(str); } - -template inline -TargetString utf8CvrtTo(const SourceString& str) -{ - return utf8CvrtTo(str, - typename GetCharType::Type(), - typename GetCharType::Type()); -} -} - -#endif //STRING_UTF8_HEADER_01832479146991573473545 diff --git a/zen/win_ver.h b/zen/win_ver.h index c2162c61..3a6f23c1 100644 --- a/zen/win_ver.h +++ b/zen/win_ver.h @@ -7,7 +7,7 @@ #ifndef WINDOWS_VERSION_HEADER_238470348254325 #define WINDOWS_VERSION_HEADER_238470348254325 -#include "win.h" +#include //includes "windows.h" namespace zen { @@ -44,8 +44,7 @@ bool winXyOrLater(DWORD major, DWORD minor) OSVERSIONINFO osvi = {}; osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); if (::GetVersionEx(&osvi)) //38 ns per call! (yes, that's nano!) -> we do NOT miss C++11 thread safe statics right now... - return osvi.dwMajorVersion > major || - (osvi.dwMajorVersion == major && osvi.dwMinorVersion >= minor); + return osvi.dwMajorVersion != major ? osvi.dwMajorVersion > major : osvi.dwMinorVersion >= minor; return false; } } diff --git a/zen/zstring.h b/zen/zstring.h index 8f7486b0..94d5b1a4 100644 --- a/zen/zstring.h +++ b/zen/zstring.h @@ -104,7 +104,7 @@ template