path: root/zen/FindFilePlus/find_file_plus.cpp

// **************************************************************************
// * This file is part of the FreeFileSync project. It is distributed under *
// * GNU General Public License: http://www.gnu.org/licenses/gpl.html       *
// * Copyright (C) Zenju (zenju AT gmx DOT de) - All Rights Reserved        *
// **************************************************************************

#include "find_file_plus.h"
#include "init_dll_binding.h"
//#include <windows.h>  //these two don't play nice with each other
#include "load_dll.h"
#include <new>

using namespace dll;
using namespace findplus;


namespace
{
struct NtFileError //exception class
{
    NtFileError(NTSTATUS errorCode) : ntError(errorCode) {}
    NTSTATUS ntError;
};


//--------------------------------------------------------------------------------------------------------------
typedef NTSTATUS (NTAPI* NtOpenFileFunc)(PHANDLE fileHandle,
                                         ACCESS_MASK desiredAccess,
                                         POBJECT_ATTRIBUTES objectAttributes,
                                         PIO_STATUS_BLOCK ioStatusBlock,
                                         ULONG shareAccess,
                                         ULONG openOptions);

typedef NTSTATUS (NTAPI* NtCloseFunc)(HANDLE handle);

typedef NTSTATUS (NTAPI* NtQueryDirectoryFileFunc)(HANDLE fileHandle,
                                                   HANDLE event,
                                                   PIO_APC_ROUTINE apcRoutine,
                                                   PVOID apcContext,
                                                   PIO_STATUS_BLOCK ioStatusBlock,
                                                   PVOID fileInformation,
                                                   ULONG length,
                                                   FILE_INFORMATION_CLASS fileInformationClass,
                                                   BOOLEAN ReturnSingleEntry,
                                                   PUNICODE_STRING fileMask,
                                                   BOOLEAN restartScan);

typedef ULONG (NTAPI* RtlNtStatusToDosErrorFunc)(NTSTATUS /*__in status*/);

typedef struct _RTL_RELATIVE_NAME_U
{
    UNICODE_STRING RelativeName;
    HANDLE ContainingDirectory;
    PVOID /*PRTLP_CURDIR_REF*/ CurDirRef;
} RTL_RELATIVE_NAME_U, *PRTL_RELATIVE_NAME_U;

typedef BOOLEAN (NTAPI* RtlDosPathNameToNtPathName_UFunc)(PCWSTR dosFileName, 	              //__in
                                                          PUNICODE_STRING ntFileName,         //__out
                                                          PCWSTR* filePart, 	              //__out
                                                          PRTL_RELATIVE_NAME_U relativeName); //__out

typedef BOOLEAN (NTAPI* RtlDosPathNameToRelativeNtPathName_UFunc)(PCWSTR dosFileName, 	              //__in
                                                                  PUNICODE_STRING ntFileName,         //__out
                                                                  PCWSTR* filePart, 	              //__out
                                                                  PRTL_RELATIVE_NAME_U relativeName); //__out

typedef BOOLEAN (NTAPI*  RtlCreateUnicodeStringFunc)(PUNICODE_STRING DestinationString, //_Out_
                                                     PCWSTR SourceString);              //_In_

typedef VOID (NTAPI* RtlFreeUnicodeStringFunc)(PUNICODE_STRING); //__inout unicodeString

//--------------------------------------------------------------------------------------------------------------

//it seems we cannot use any of the ntoskrnl.lib files in WinDDK as they produce access violations
//fortunately dynamic binding works fine:
const SysDllFun<NtOpenFileFunc>			     	  ntOpenFile                  (L"ntdll.dll", "NtOpenFile");
const SysDllFun<NtCloseFunc>				      ntClose                     (L"ntdll.dll", "NtClose");
const SysDllFun<NtQueryDirectoryFileFunc>	      ntQueryDirectoryFile        (L"ntdll.dll", "NtQueryDirectoryFile");
const SysDllFun<RtlNtStatusToDosErrorFunc>	      rtlNtStatusToDosError       (L"ntdll.dll", "RtlNtStatusToDosError");
const SysDllFun<RtlCreateUnicodeStringFunc>	      rtlCreateUnicodeString      (L"ntdll.dll", "RtlCreateUnicodeString");
const SysDllFun<RtlFreeUnicodeStringFunc>         rtlFreeUnicodeString        (L"ntdll.dll", "RtlFreeUnicodeString");
const SysDllFun<RtlDosPathNameToNtPathName_UFunc> rtlDosPathNameToNtPathName_U(SysDllFun<RtlDosPathNameToRelativeNtPathName_UFunc>(L"ntdll.dll", "RtlDosPathNameToRelativeNtPathName_U") ?
                                                                               SysDllFun<RtlDosPathNameToRelativeNtPathName_UFunc>(L"ntdll.dll", "RtlDosPathNameToRelativeNtPathName_U") : //use the newer version if available
                                                                               SysDllFun<RtlDosPathNameToNtPathName_UFunc>(L"ntdll.dll", "RtlDosPathNameToNtPathName_U")); //fallback for XP
//global constants only -> preserve thread safety!
}


bool findplus::initDllBinding() //evaluate in ::DllMain() when attaching process
{
    //NT/ZwXxx Routines
    //http://msdn.microsoft.com/en-us/library/ff567122(v=VS.85).aspx

    //Run-Time Library (RTL) Routines
    //http://msdn.microsoft.com/en-us/library/ff563638(v=VS.85).aspx

    //verify dynamic dll binding
    return ntOpenFile             &&
           ntClose                &&
           ntQueryDirectoryFile   &&
           rtlNtStatusToDosError  &&
           rtlCreateUnicodeString &&
           rtlFreeUnicodeString   &&
           rtlDosPathNameToNtPathName_U;

    //this may become handy some time: nt status code STATUS_ORDINAL_NOT_FOUND maps to win32 code ERROR_INVALID_ORDINAL
}


class findplus::FileSearcher
{
public:
    FileSearcher(const wchar_t* dirname); //throw NtFileError
    ~FileSearcher();

    bool readDir(FileInformation& output); //throw NtFileError; returns false if "no more files"

private:
    template <class QueryPolicy> bool readDirImpl(FileInformation& output); //throw NtFileError

    UNICODE_STRING dirnameNt; //it seems hDir implicitly keeps a reference to this, at least ::FindFirstFile() does no cleanup before ::FindClose()!
    HANDLE hDir;

    ULONG  nextEntryOffset; //!= 0 if entry is waiting in buffer
    //::FindNextFileW() uses 0x1000 = 4096 = sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * 2000
    //=> let's use the same, even if our header is 16 byte larger; maybe there is some packet size advantage for networks? Note that larger buffers seem to degrade performance.
    static const ULONG BUFFER_SIZE = 4096;
    LONGLONG buffer[BUFFER_SIZE / sizeof(LONGLONG)]; //buffer needs to be aligned at LONGLONG boundary

    static_assert(BUFFER_SIZE % sizeof(LONGLONG) == 0, "ups, our buffer is trimmed!");
};


//a simple scope guard without <utility>, <type_traits>, <cassert> dependencies:
template <typename F>
class ScopeGuardLean
{
public:
    explicit ScopeGuardLean(F fun) : dismissed_(false), fun_(fun) {}
    ScopeGuardLean(ScopeGuardLean&& other) : dismissed_(other.dismissed_), fun_(std::move(other.fun_)) { other.dismiss(); }

    void dismiss() { dismissed_ = true; }

    ~ScopeGuardLean()
    {
        if (!dismissed_)
            try { fun_(); }
            catch (...) {}
    }

private:
    ScopeGuardLean           (const ScopeGuardLean&); // = delete
    ScopeGuardLean& operator=(const ScopeGuardLean&); //

    bool dismissed_;
    F fun_;
};

template <class F> inline
ScopeGuardLean<F> makeGuard(F fun) { return ScopeGuardLean<F>(fun); }


FileSearcher::FileSearcher(const wchar_t* dirname) :
    dirnameNt(), //[!]
    hDir(nullptr),
    nextEntryOffset(0)
{
    auto guardConstructor = makeGuard([&] { this->~FileSearcher(); });
    //--------------------------------------------------------------------------------------------------------------

    //convert dosFileName, e.g. C:\Users or \\?\C:\Users to ntFileName \??\C:\Users
    //in contrast to ::FindFirstFile() implementation we don't evaluate the relativeName,
    //however tests indicate ntFileName is *always* filled with an absolute name, even if dosFileName is relative

    PCWSTR filePart = nullptr;
    RTL_RELATIVE_NAME_U relativeName = {};

    //NOTE: RtlDosPathNameToNtPathName_U may be used on all XP/Win7/Win8 for compatibility
    //		RtlDosPathNameToNtPathName_U:         used by Windows XP          available with OS version 3.51 (Windows NT) and higher
    //		RtlDosPathNameToRelativeNtPathName_U: used by Win7/Win8           available with OS version 5.2  (Windows Server 2003) and higher
    if (!rtlDosPathNameToNtPathName_U(dirname,        //__in dosFileName,
                                      &dirnameNt,     //__out ntFileName,
                                      &filePart,      //__out FilePart     - points into ntFileName
                                      &relativeName)) //__out relativeName - points into ntFileName; empty if dosFileName is absolute
        throw NtFileError(STATUS_OBJECT_PATH_NOT_FOUND); //translates to ERROR_PATH_NOT_FOUND, same behavior like ::FindFirstFileEx()
    //note 1: internally it distinguishes between "quick path" == \\?\ and "slow path" handling!
    //http://doxygen.reactos.org/d9/d6e/lib_2rtl_2path_8c_a11c87ad0f7752999b0b8972af6165d7a.html#a11c87ad0f7752999b0b8972af6165d7a

    //note 2: without \\?\, i.e. slow path handling it calls RtlGetFullPathName_Ustr() which removes trailing spaces!!!
    //http://doxygen.reactos.org/d9/d6e/lib_2rtl_2path_8c_a8624b864678ca64b031f5fc273e022af.html#a8624b864678ca64b031f5fc273e022af
    //FindFirstFile() gets lucky because it passes "<dirname>\*" which never has trailing space chars! >:(

    OBJECT_ATTRIBUTES objAttr = {};
    if (relativeName.RelativeName.Length == 0) //absolute name
    {
        InitializeObjectAttributes(&objAttr,             //[out] POBJECT_ATTRIBUTES initializedAttributes,
                                   &dirnameNt,           //[in]  PUNICODE_STRING objectName,
                                   OBJ_CASE_INSENSITIVE, //[in]  ULONG attributes,
                                   nullptr,              //[in]  HANDLE rootDirectory,
                                   nullptr);             //[in, optional] PSECURITY_DESCRIPTOR securityDescriptor
    }
    else //relative name (it seems we alternatively could also use dirnameNt here?)
    {
        InitializeObjectAttributes(&objAttr,                         //[out] POBJECT_ATTRIBUTES initializedAttributes,
                                   &relativeName.RelativeName,       //[in]  PUNICODE_STRING objectName,
                                   OBJ_CASE_INSENSITIVE,             //[in]  ULONG attributes,
                                   relativeName.ContainingDirectory, //[in]  HANDLE rootDirectory,
                                   nullptr);                         //[in, optional] PSECURITY_DESCRIPTOR securityDescriptor
    }

    {
        IO_STATUS_BLOCK status = {};
        NTSTATUS rv = ntOpenFile(&hDir,                             //__out PHANDLE FileHandle,
                                 FILE_LIST_DIRECTORY | SYNCHRONIZE, //__in ACCESS_MASK desiredAccess, - 100001 used by ::FindFirstFile() on all XP/Win7/Win8
                                 &objAttr,                          //__in POBJECT_ATTRIBUTES objectAttributes,
                                 &status,                           //__out PIO_STATUS_BLOCK ioStatusBlock,
                                 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, //__in ULONG shareAccess, - 7 on Win7/Win8, 3 on XP
                                 FILE_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT | FILE_OPEN_FOR_BACKUP_INTENT); //__in ULONG openOptions - 4021 used on all XP/Win7/Win8
        if (!NT_SUCCESS(rv))
            throw NtFileError(rv);
    }

    guardConstructor.dismiss();
}


inline
FileSearcher::~FileSearcher()
{
    //cleanup in reverse order
    if (hDir)
        ntClose(hDir);

    if (dirnameNt.Buffer)
        rtlFreeUnicodeString(&dirnameNt); //cleanup identical to ::FindFirstFile()
    //note that most of this function seems inlined by the linker, so that its assembly looks equivalent to "RtlFreeHeap(RtlGetProcessHeap(), 0, ntPathName.Buffer)"
}


namespace
{
/*
Common C-style policy handling directory traversal:

struct QueryPolicy
{
	typedef ... RawFileInfo;
	static const FILE_INFORMATION_CLASS fileInformationClass = ...;
	static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo);
};
*/

struct DirQueryDefault //as implemented in Win32 FindFirstFile()/FindNextFile()
{
    typedef FILE_BOTH_DIR_INFORMATION RawFileInfo;
    static const FILE_INFORMATION_CLASS fileInformationClass = FileBothDirectoryInformation;
    static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo) { fileInfo.fileId.QuadPart = 0; }
};

struct DirQueryFileId
{
    typedef FILE_ID_BOTH_DIR_INFORMATION RawFileInfo;
    static const FILE_INFORMATION_CLASS fileInformationClass = FileIdBothDirectoryInformation;
    static void extractFileId(const RawFileInfo& rawInfo, FileInformation& fileInfo)
    {
        fileInfo.fileId.QuadPart = rawInfo.FileId.QuadPart; //may be 0 even in this context, e.g. for mapped FTP drive!
        static_assert(sizeof(fileInfo.fileId) == sizeof(rawInfo.FileId), "dang!");
    }
};
}


inline
bool FileSearcher::readDir(FileInformation& output) { return readDirImpl<DirQueryFileId>(output); } //throw NtFileError


template <class QueryPolicy>
bool FileSearcher::readDirImpl(FileInformation& output) //throw NtFileError; returns false if "no more files"
{
    //although FILE_ID_FULL_DIR_INFORMATION should suffice for our purposes, there are problems on Windows XP for certain directories, e.g. "\\Vboxsvr\build"
    //making NtQueryDirectoryFile() return with STATUS_INVALID_PARAMETER while other directories, e.g. "C:\" work fine for some reason
    //FILE_ID_BOTH_DIR_INFORMATION on the other hand works on XP/Win7/Win8
    //performance: there is no noticeable difference between FILE_ID_BOTH_DIR_INFORMATION and FILE_ID_FULL_DIR_INFORMATION

    /* corresponding first access in ::FindFirstFileW()

    NTSTATUS rv = ntQueryDirectoryFile(hDir,        //__in HANDLE fileHandle,
                                       nullptr,        //__in_opt HANDLE event,
                                       nullptr,        //__in_opt PIO_APC_ROUTINE apcRoutine,
                                       nullptr,        //__in_opt PVOID apcContext,
                                       &status,     //__out PIO_STATUS_BLOCK ioStatusBlock,
                                       &buffer,     //__out_bcount(Length) PVOID fileInformation,
                                       BUFFER_SIZE, //__in ULONG length, ::FindFirstFileW() on all XP/Win7/Win8 uses sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * MAX_PATH == 0x268
                                       FileIdBothDirectoryInformation, //__in FILE_INFORMATION_CLASS fileInformationClass - all XP/Win7/Win8 use "FileBothDirectoryInformation"
                                       true,        //__in BOOLEAN returnSingleEntry,
                                       nullptr,        //__in_opt PUNICODE_STRING mask,
                                       false);      //__in BOOLEAN restartScan
    */

    //analog to ::FindNextFileW() with performance optimized access (in contrast to first access in ::FindFirstFileW())
    if (nextEntryOffset == 0)
    {
        IO_STATUS_BLOCK status = {};
        NTSTATUS rv = ntQueryDirectoryFile(hDir,        //__in HANDLE fileHandle,
                                           nullptr,     //__in_opt HANDLE event,
                                           nullptr,     //__in_opt PIO_APC_ROUTINE apcRoutine,
                                           nullptr,     //__in_opt PVOID apcContext,
                                           &status,     //__out PIO_STATUS_BLOCK ioStatusBlock,
                                           &buffer,     //__out_bcount(Length) PVOID fileInformation,
                                           BUFFER_SIZE, //__in ULONG length, ::FindNextFileW() on all XP/Win7/Win8 uses sizeof(FILE_BOTH_DIR_INFORMATION) + sizeof(TCHAR) * 2000 == 0x1000
                                           QueryPolicy::fileInformationClass, //__in FILE_INFORMATION_CLASS fileInformationClass - all XP/Win7/Win8 use "FileBothDirectoryInformation"
                                           false,       //__in BOOLEAN returnSingleEntry,
                                           nullptr,     //__in_opt PUNICODE_STRING mask,
                                           false);      //__in BOOLEAN restartScan
        if (!NT_SUCCESS(rv))
        {
            /*
            fallback to default directory query method, if FileIdBothDirectoryInformation is not properly implemented
            this is required for NetDrive mounted Webdav, e.g. www.box.net and NT4, 2000 remote drives, et al.

            NT status code                  |  Win32 error code
            --------------------------------|------------------------
            STATUS_INVALID_LEVEL            | ERROR_INVALID_LEVEL
            STATUS_NOT_SUPPORTED            | ERROR_NOT_SUPPORTED
            STATUS_UNEXPECTED_NETWORK_ERROR | ERROR_UNEXP_NET_ERR -> traverse network drive hosted by Win98
            STATUS_INVALID_PARAMETER        | ERROR_INVALID_PARAMETER
            STATUS_INVALID_NETWORK_RESPONSE | ERROR_BAD_NET_RESP
            STATUS_INVALID_INFO_CLASS       | ERROR_INVALID_PARAMETER
            STATUS_UNSUCCESSFUL             | ERROR_GEN_FAILURE
            STATUS_ACCESS_VIOLATION         | ERROR_NOACCESS      ->FileIdBothDirectoryInformation on XP accessing UDF
            STATUS_NO_SUCH_FILE             | ERROR_FILE_NOT_FOUND

            rv == STATUS_NO_SUCH_FILE:
            failure to find a file on first call returns STATUS_NO_SUCH_FILE, while subsequent accesses return STATUS_NO_MORE_FILES
            note: not all directories contain ".", ".." entries! E.g. a drive's root directory or NetDrive + ftp.gnu.org\CRYPTO.README"
            -> addon: this is NOT a directory, it looks like one in NetDrive, but it's a file in Opera
            STATUS_NO_SUCH_FILE is abused by some citrix shares instead of "STATUS_INVALID_PARAMETER" so we treat it as such!
            => since the directory is "truly empty" a fallback won't hurt
            */
            if (rv == STATUS_NO_MORE_FILES) //perf: don't throw an exception in this common case! => 8% perf boost for FFS comparison phase!
                return false;

            if (rv == STATUS_NOT_SUPPORTED            ||
                rv == STATUS_INVALID_LEVEL            ||
                rv == STATUS_NO_SUCH_FILE             || //[!]
                rv == STATUS_UNEXPECTED_NETWORK_ERROR ||
                rv == STATUS_INVALID_PARAMETER        ||
                rv == STATUS_INVALID_NETWORK_RESPONSE ||
                rv == STATUS_INVALID_INFO_CLASS       ||
                //rv == STATUS_NOT_IMPLEMENTED          || -> first confirm that these codes
                //rv == STATUS_INVALID_DEVICE_REQUEST   || -> are in fact used!
                rv == STATUS_UNSUCCESSFUL             ||
                rv == STATUS_ACCESS_VIOLATION)
                rv = STATUS_NOT_SUPPORTED;

            throw NtFileError(rv); //throws STATUS_NO_MORE_FILES when finished
        }

        //  for (NTSTATUS i = 0xC0000000L; i != -1; ++i)
        //  {
        //      if (rtlNtStatusToDosError(i) == 59) //ERROR_UNEXP_NET_ERR
        //		    __debugbreak(); //__asm int 3;
        //	}

        if (status.Information == 0)                   //except for the first call to call ::NtQueryDirectoryFile():
            throw NtFileError(STATUS_BUFFER_OVERFLOW); //if buffer size is too small, return value is STATUS_SUCCESS and Information == 0 -> we don't expect this!
    }

    typedef typename QueryPolicy::RawFileInfo RawFileInfo;
    const RawFileInfo& dirInfo = *reinterpret_cast<RawFileInfo*>(reinterpret_cast<char*>(buffer) + nextEntryOffset);

    if (dirInfo.NextEntryOffset == 0)
        nextEntryOffset = 0; //our offset is relative to the beginning of the buffer
    else
        nextEntryOffset += dirInfo.NextEntryOffset;


    auto toFileTime = [](const LARGE_INTEGER& rawTime) -> FILETIME
    {
        FILETIME tmp = { rawTime.LowPart, rawTime.HighPart };
        return tmp;
    };

    QueryPolicy::extractFileId(dirInfo, output);

    output.creationTime      = toFileTime(dirInfo.CreationTime);
    output.lastWriteTime     = toFileTime(dirInfo.LastWriteTime); //the similar field "ChangeTime" refers to changes to metadata in addition to write accesses
    output.fileSize.QuadPart = dirInfo.EndOfFile.QuadPart;
    output.fileAttributes    = dirInfo.FileAttributes;
    output.shortNameLength   = dirInfo.FileNameLength / sizeof(TCHAR); //FileNameLength is in bytes!

    if (dirInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) //analog to FindFirstFile(), confirmed for Win XP and Win 7
        output.reparseTag = dirInfo.EaSize;                    //
    else
        output.reparseTag = 0;

    if (dirInfo.FileNameLength + sizeof(TCHAR) > sizeof(output.shortName)) //this may actually happen if ::NtQueryDirectoryFile() decides to return a
        throw NtFileError(STATUS_BUFFER_OVERFLOW);    //short name of length MAX_PATH + 1, 0-termination is not required!

    ::memcpy(output.shortName, dirInfo.FileName, dirInfo.FileNameLength);
    output.shortName[output.shortNameLength] = 0; //NOTE: FILE_ID_BOTH_DIR_INFORMATION::FileName in general is NOT 0-terminated! It is on XP/Win7, but NOT on Win8!

    static_assert(sizeof(output.creationTime)   == sizeof(dirInfo.CreationTime),   "dang!");
    static_assert(sizeof(output.lastWriteTime)  == sizeof(dirInfo.LastWriteTime),  "dang!");
    static_assert(sizeof(output.fileSize)       == sizeof(dirInfo.EndOfFile),      "dang!");
    static_assert(sizeof(output.fileAttributes) == sizeof(dirInfo.FileAttributes), "dang!");
    return true;
}


FindHandle findplus::openDir(const wchar_t* dirname)
{
    try
    {
        return new FileSearcher(dirname); //throw NtFileError, std::bad_alloc
    }
    catch (const NtFileError& e)
    {
        setWin32Error(rtlNtStatusToDosError(e.ntError));
        return nullptr;
    }
    catch (const std::bad_alloc&) //not unlikely in file sync context! => handle!
    {
        setWin32Error(rtlNtStatusToDosError(STATUS_NO_MEMORY));
        return nullptr;
    }
}


bool findplus::readDir(FindHandle hnd, FileInformation& output)
{
    try
    {
        if (!hnd->readDir(output)) //throw NtFileError
        {
            setWin32Error(rtlNtStatusToDosError(STATUS_NO_MORE_FILES));
            return false;
        }
        return true;
    }
    catch (const NtFileError& e)
    {
        setWin32Error(rtlNtStatusToDosError(e.ntError));
        return false;
    }
}


void findplus::closeDir(FindHandle hnd)
{
    delete hnd;
}