// ************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) ZenJu (zhnmju123 AT gmx DOT de) - All Rights Reserved * // ************************************************************************** #include "find_file_plus.h" #include "init_dll_binding.h" //#include //these two don't play nice with each other #include "load_dll.h" #include #include using namespace dll; using namespace findplus; namespace { struct NtFileError //exception class { NtFileError(NTSTATUS errorCode) : ntError(errorCode) {} NTSTATUS ntError; }; //-------------------------------------------------------------------------------------------------------------- typedef NTSTATUS (NTAPI* NtOpenFileFunc)(PHANDLE fileHandle, ACCESS_MASK desiredAccess, POBJECT_ATTRIBUTES objectAttributes, PIO_STATUS_BLOCK ioStatusBlock, ULONG shareAccess, ULONG openOptions); typedef NTSTATUS (NTAPI* NtCloseFunc)(HANDLE handle); typedef NTSTATUS (NTAPI* NtQueryDirectoryFileFunc)(HANDLE fileHandle, HANDLE event, PIO_APC_ROUTINE apcRoutine, PVOID apcContext, PIO_STATUS_BLOCK ioStatusBlock, PVOID fileInformation, ULONG length, FILE_INFORMATION_CLASS fileInformationClass, BOOLEAN ReturnSingleEntry, PUNICODE_STRING fileMask, BOOLEAN restartScan); typedef ULONG (NTAPI* RtlNtStatusToDosErrorFunc)(NTSTATUS /*__in status*/); typedef struct _RTL_RELATIVE_NAME_U { UNICODE_STRING RelativeName; HANDLE ContainingDirectory; PVOID /*PRTLP_CURDIR_REF*/ CurDirRef; } RTL_RELATIVE_NAME_U, *PRTL_RELATIVE_NAME_U; typedef BOOLEAN (NTAPI* RtlDosPathNameToNtPathName_UFunc)(PCWSTR, //__in dosFileName, PUNICODE_STRING, //__out ntFileName, PCWSTR*, //__out_optFilePart, PRTL_RELATIVE_NAME_U); //__out_opt relativeName typedef BOOLEAN (NTAPI* RtlDosPathNameToRelativeNtPathName_UFunc)(PCWSTR, //__in dosFileName, PUNICODE_STRING, //__out ntFileName, PCWSTR*, //__out_optFilePart, PRTL_RELATIVE_NAME_U); //__out_opt relativeName typedef VOID (NTAPI* RtlFreeUnicodeStringFunc)(PUNICODE_STRING); //__inout unicodeString //-------------------------------------------------------------------------------------------------------------- //it seems we cannot use any of the ntoskrnl.lib files in WinDDK as they produce access violations //fortunately dynamic binding works fine: const SysDllFun ntOpenFile (L"ntdll.dll", "NtOpenFile"); const SysDllFun ntClose (L"ntdll.dll", "NtClose"); const SysDllFun ntQueryDirectoryFile (L"ntdll.dll", "NtQueryDirectoryFile"); const SysDllFun rtlNtStatusToDosError (L"ntdll.dll", "RtlNtStatusToDosError"); const SysDllFun rtlFreeUnicodeString (L"ntdll.dll", "RtlFreeUnicodeString"); const SysDllFun rtlDosPathNameToNtPathName_U(SysDllFun(L"ntdll.dll", "RtlDosPathNameToRelativeNtPathName_U") ? SysDllFun(L"ntdll.dll", "RtlDosPathNameToRelativeNtPathName_U") : //use the newer version if available SysDllFun(L"ntdll.dll", "RtlDosPathNameToNtPathName_U")); //fallback for XP //global constants only -> preserve thread safety! } bool findplus::initDllBinding() //evaluate in ::DllMain() when attaching process { //NT/ZwXxx Routines //http://msdn.microsoft.com/en-us/library/ff567122(v=VS.85).aspx //Run-Time Library (RTL) Routines //http://msdn.microsoft.com/en-us/library/ff563638(v=VS.85).aspx //verify dynamic dll binding return ntOpenFile && ntClose && ntQueryDirectoryFile && rtlNtStatusToDosError && rtlFreeUnicodeString && rtlDosPathNameToNtPathName_U; //this may become handy some time: nt status code STATUS_ORDINAL_NOT_FOUND maps to win32 code ERROR_INVALID_ORDINAL } class findplus::FileSearcher { public: FileSearcher(const wchar_t* dirname); //throw FileError ~FileSearcher(); void readDir(FileInformation& output); //throw FileError private: template void readDirImpl(FileInformation& output); //throw FileError UNICODE_STRING dirnameNt; //it seems hDir implicitly keeps a reference to this, at least ::FindFirstFile() does no cleanup before ::FindClose()! HANDLE hDir; ULONG nextEntryOffset; //!= 0 if entry is waiting in buffer //::FindNextFileW() uses 0x1000 = 4096 = sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * 2000 //=> let's use the same, even if our header is 16 byte larger; maybe there is some packet size advantage for networks? Note that larger buffers seem to degrade performance. static const ULONG BUFFER_SIZE = 4096; LONGLONG buffer[BUFFER_SIZE / sizeof(LONGLONG)]; //buffer needs to be aligned at LONGLONG boundary static_assert(BUFFER_SIZE % sizeof(LONGLONG) == 0, "ups, our buffer is trimmed!"); }; FileSearcher::FileSearcher(const wchar_t* dirname) : hDir(nullptr), nextEntryOffset(0) { dirnameNt.Buffer = nullptr; dirnameNt.Length = 0; dirnameNt.MaximumLength = 0; zen::ScopeGuard guardConstructor = zen::makeGuard([&]() { this->~FileSearcher(); }); //-------------------------------------------------------------------------------------------------------------- //convert dosFileName, e.g. C:\Users or \\?\C:\Users to ntFileName \??\C:\Users //in contrast to ::FindFirstFile() implementation we don't evaluate the relativeName, //however tests indicate ntFileName is *always* filled with an absolute name, even if dosFileName is relative //NOTE: RtlDosPathNameToNtPathName_U may be used on all XP/Win7/Win8 for compatibility // RtlDosPathNameToNtPathName_U: used by Windows XP available with OS version 3.51 (Windows NT) and higher // RtlDosPathNameToRelativeNtPathName_U: used by Win7/Win8 available with OS version 5.2 (Windows Server 2003) and higher if (!rtlDosPathNameToNtPathName_U(dirname, //__in dosFileName, &dirnameNt, //__out ntFileName, nullptr, //__out_optFilePart, nullptr)) //__out_opt relativeName - empty if dosFileName is absolute throw NtFileError(STATUS_OBJECT_PATH_NOT_FOUND); //translates to ERROR_PATH_NOT_FOUND, same behavior like ::FindFirstFileEx() OBJECT_ATTRIBUTES objAttr = {}; InitializeObjectAttributes(&objAttr, //[out] POBJECT_ATTRIBUTES initializedAttributes, &dirnameNt, //[in] PUNICODE_STRING objectName, OBJ_CASE_INSENSITIVE, //[in] ULONG attributes, nullptr, //[in] HANDLE rootDirectory, nullptr); //[in, optional] PSECURITY_DESCRIPTOR securityDescriptor { IO_STATUS_BLOCK status = {}; NTSTATUS rv = ntOpenFile(&hDir, //__out PHANDLE FileHandle, FILE_LIST_DIRECTORY | SYNCHRONIZE, //__in ACCESS_MASK desiredAccess, - 100001 used by ::FindFirstFile() on all XP/Win7/Win8 &objAttr, //__in POBJECT_ATTRIBUTES objectAttributes, &status, //__out PIO_STATUS_BLOCK ioStatusBlock, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //__in ULONG shareAccess, - 7 on Win7/Win8, 3 on XP FILE_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT | FILE_OPEN_FOR_BACKUP_INTENT); //__in ULONG openOptions - 4021 used on all XP/Win7/Win8 if (!NT_SUCCESS(rv)) throw NtFileError(rv); } guardConstructor.dismiss(); } inline FileSearcher::~FileSearcher() { //cleanup in reverse order if (hDir) ntClose(hDir); if (dirnameNt.Buffer) rtlFreeUnicodeString(&dirnameNt); //cleanup identical to ::FindFirstFile() //note that most of this function seems inlined by the linker, so that its assembly looks equivalent to "RtlFreeHeap(GetProcessHeap(), 0, ntPathName.Buffer)" } namespace { /* Common C-style policy handling directory traversal: struct QueryPolicy { typedef ... RawFileInfo; static const FILE_INFORMATION_CLASS fileInformationClass = ...; static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo); }; */ struct DirQueryDefault //as implemented in Win32 FindFirstFile()/FindNextFile() { typedef FILE_BOTH_DIR_INFORMATION RawFileInfo; static const FILE_INFORMATION_CLASS fileInformationClass = FileBothDirectoryInformation; static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo) { fileInfo.fileId.QuadPart = 0; } }; struct DirQueryFileId { typedef FILE_ID_BOTH_DIR_INFORMATION RawFileInfo; static const FILE_INFORMATION_CLASS fileInformationClass = FileIdBothDirectoryInformation; static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo) { fileInfo.fileId.QuadPart = rawInfo.FileId.QuadPart; //may be 0 even in this context, e.g. for mapped FTP drive! static_assert(sizeof(fileInfo.fileId) == sizeof(rawInfo.FileId), "dang!"); } }; } inline void FileSearcher::readDir(FileInformation& output) { readDirImpl(output); } //throw FileError template void FileSearcher::readDirImpl(FileInformation& output) //throw FileError { //although FILE_ID_FULL_DIR_INFORMATION should suffice for our purposes, there are problems on Windows XP for certain directories, e.g. "\\Vboxsvr\build" //making NtQueryDirectoryFile() return with STATUS_INVALID_PARAMETER while other directories, e.g. "C:\" work fine for some reason //FILE_ID_BOTH_DIR_INFORMATION on the other hand works on XP/Win7/Win8 //performance: there is no noticeable difference between FILE_ID_BOTH_DIR_INFORMATION and FILE_ID_FULL_DIR_INFORMATION /* corresponding first access in ::FindFirstFileW() NTSTATUS rv = ntQueryDirectoryFile(hDir, //__in HANDLE fileHandle, nullptr, //__in_opt HANDLE event, nullptr, //__in_opt PIO_APC_ROUTINE apcRoutine, nullptr, //__in_opt PVOID apcContext, &status, //__out PIO_STATUS_BLOCK ioStatusBlock, &buffer, //__out_bcount(Length) PVOID fileInformation, BUFFER_SIZE, //__in ULONG length, ::FindFirstFileW() on all XP/Win7/Win8 uses sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * MAX_PATH == 0x268 FileIdBothDirectoryInformation, //__in FILE_INFORMATION_CLASS fileInformationClass - all XP/Win7/Win8 use "FileBothDirectoryInformation" true, //__in BOOLEAN returnSingleEntry, nullptr, //__in_opt PUNICODE_STRING mask, false); //__in BOOLEAN restartScan */ //analog to ::FindNextFileW() with performance optimized access (in contrast to first access in ::FindFirstFileW()) if (nextEntryOffset == 0) { IO_STATUS_BLOCK status = {}; NTSTATUS rv = ntQueryDirectoryFile(hDir, //__in HANDLE fileHandle, nullptr, //__in_opt HANDLE event, nullptr, //__in_opt PIO_APC_ROUTINE apcRoutine, nullptr, //__in_opt PVOID apcContext, &status, //__out PIO_STATUS_BLOCK ioStatusBlock, &buffer, //__out_bcount(Length) PVOID fileInformation, BUFFER_SIZE, //__in ULONG length, ::FindNextFileW() on all XP/Win7/Win8 uses sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * 2000 == 0x1000 QueryPolicy::fileInformationClass, //__in FILE_INFORMATION_CLASS fileInformationClass - all XP/Win7/Win8 use "FileBothDirectoryInformation" false, //__in BOOLEAN returnSingleEntry, nullptr, //__in_opt PUNICODE_STRING mask, false); //__in BOOLEAN restartScan if (!NT_SUCCESS(rv)) { /* fallback to default directory query method, if FileIdBothDirectoryInformation is not properly implemented this is required for NetDrive mounted Webdav, e.g. www.box.net and NT4, 2000 remote drives, et al. NT status code | Win32 error code --------------------------------|------------------------ STATUS_INVALID_LEVEL | ERROR_INVALID_LEVEL STATUS_NOT_SUPPORTED | ERROR_NOT_SUPPORTED STATUS_UNEXPECTED_NETWORK_ERROR | ERROR_UNEXP_NET_ERR -> traverse network drive hosted by Win98 STATUS_INVALID_PARAMETER | ERROR_INVALID_PARAMETER STATUS_INVALID_NETWORK_RESPONSE | ERROR_BAD_NET_RESP STATUS_INVALID_INFO_CLASS | ERROR_INVALID_PARAMETER STATUS_UNSUCCESSFUL | ERROR_GEN_FAILURE STATUS_ACCESS_VIOLATION | ERROR_NOACCESS ->FileIdBothDirectoryInformation on XP accessing UDF STATUS_NO_SUCH_FILE | ERROR_FILE_NOT_FOUND rv == STATUS_NO_SUCH_FILE: failure to find a file on first call returns STATUS_NO_SUCH_FILE, while subsequent accesses return STATUS_NO_MORE_FILES note: not all directories contain ".", ".." entries! E.g. a drive's root directory or NetDrive + ftp.gnu.org\CRYPTO.README" -> addon: this is NOT a directory, it looks like one in NetDrive, but it's a file in Opera STATUS_NO_SUCH_FILE is abused by some citrix shares instead of "STATUS_INVALID_PARAMETER" so we treat it as such! => since the directory is "truly empty" a fallback won't hurt */ if (rv == STATUS_INVALID_LEVEL || rv == STATUS_NOT_SUPPORTED || rv == STATUS_UNEXPECTED_NETWORK_ERROR || rv == STATUS_INVALID_PARAMETER || rv == STATUS_INVALID_NETWORK_RESPONSE || rv == STATUS_INVALID_INFO_CLASS || rv == STATUS_UNSUCCESSFUL || rv == STATUS_ACCESS_VIOLATION || rv == STATUS_NO_SUCH_FILE) rv = STATUS_NOT_SUPPORTED; throw NtFileError(rv); //throws STATUS_NO_MORE_FILES when finished } //for (NTSTATUS i = 0xC0000000L; i != -1; ++i) //{ // if (rtlNtStatusToDosError(i) == 59) //ERROR_UNEXP_NET_ERR // __asm int 3; //} if (status.Information == 0) //except for the first call to call ::NtQueryDirectoryFile(): throw NtFileError(STATUS_BUFFER_OVERFLOW); //if buffer size is too small, return value is STATUS_SUCCESS and Information == 0 -> we don't expect this! } typedef typename QueryPolicy::RawFileInfo RawFileInfo; const RawFileInfo& dirInfo = *reinterpret_cast(reinterpret_cast(buffer) + nextEntryOffset); if (dirInfo.NextEntryOffset == 0) nextEntryOffset = 0; //our offset is relative to the beginning of the buffer else nextEntryOffset += dirInfo.NextEntryOffset; auto toFileTime = [](const LARGE_INTEGER & rawTime) -> FILETIME { FILETIME tmp = { rawTime.LowPart, rawTime.HighPart }; return tmp; }; QueryPolicy::extractFileId(dirInfo, output); output.creationTime = toFileTime(dirInfo.CreationTime); output.lastWriteTime = toFileTime(dirInfo.LastWriteTime); //the similar field "ChangeTime" refers to changes to metadata in addition to write accesses output.fileSize.QuadPart = dirInfo.EndOfFile.QuadPart; output.fileAttributes = dirInfo.FileAttributes; output.shortNameLength = dirInfo.FileNameLength / sizeof(TCHAR); //FileNameLength is in bytes! if (dirInfo.FileNameLength + sizeof(TCHAR) > sizeof(output.shortName)) //this may actually happen if ::NtQueryDirectoryFile() decides to return a throw NtFileError(STATUS_BUFFER_OVERFLOW); //short name of length MAX_PATH + 1, 0-termination is not required! ::memcpy(output.shortName, dirInfo.FileName, dirInfo.FileNameLength); output.shortName[output.shortNameLength] = 0; //NOTE: FILE_ID_BOTH_DIR_INFORMATION::FileName in general is NOT 0-terminated! It is on XP/Win7, but NOT on Win8! static_assert(sizeof(output.creationTime) == sizeof(dirInfo.CreationTime), "dang!"); static_assert(sizeof(output.lastWriteTime) == sizeof(dirInfo.LastWriteTime), "dang!"); static_assert(sizeof(output.fileSize) == sizeof(dirInfo.EndOfFile), "dang!"); static_assert(sizeof(output.fileAttributes) == sizeof(dirInfo.FileAttributes), "dang!"); } FindHandle findplus::openDir(const wchar_t* dirname) { try { return new FileSearcher(dirname); //throw FileError } catch (const NtFileError& e) { setWin32Error(rtlNtStatusToDosError(e.ntError)); return nullptr; } } bool findplus::readDir(FindHandle hnd, FileInformation& output) { try { hnd->readDir(output); //throw FileError return true; } catch (const NtFileError& e) { setWin32Error(rtlNtStatusToDosError(e.ntError)); return false; } } void findplus::closeDir(FindHandle hnd) { delete hnd; }