4.1

1 files changed, 597 insertions, 0 deletions

diff --git a/lib/db_file.cpp b/lib/db_file.cpp
new file mode 100644
index 00000000..757a95d7
--- /dev/null
+++ b/lib/db_file.cpp
@@ -0,0 +1,597 @@
+// **************************************************************************
+// * This file is part of the FreeFileSync project. It is distributed under *
+// * GNU General Public License: http://www.gnu.org/licenses/gpl.html       *
+// * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de)                    *
+// **************************************************************************
+
+#include "db_file.h"
+#include <wx/wfstream.h>
+#include <wx/zstream.h>
+#include <wx/mstream.h>
+#include <zen/file_error.h>
+#include <wx+/string_conv.h>
+#include <zen/file_handling.h>
+#include <wx+/serialize.h>
+#include <zen/file_io.h>
+#include <zen/scope_guard.h>
+#include <zen/guid.h>
+#include <boost/bind.hpp>
+
+#ifdef FFS_WIN
+#include <zen/win.h> //includes "windows.h"
+#include <zen/long_path_prefix.h>
+#endif
+
+using namespace zen;
+
+
+namespace
+{
+//-------------------------------------------------------------------------------------------------------------------------------
+const char FILE_FORMAT_DESCR[] = "FreeFileSync";
+const int FILE_FORMAT_VER = 7;
+//-------------------------------------------------------------------------------------------------------------------------------
+
+
+template <SelectedSide side> inline
+Zstring getDBFilename(const BaseDirMapping& baseMap, bool tempfile = false)
+{
+    //Linux and Windows builds are binary incompatible: char/wchar_t case, sensitive/insensitive
+    //32 and 64 bit db files ARE designed to be binary compatible!
+    //Give db files different names.
+    //make sure they end with ".ffs_db". These files will not be included into comparison when located in base sync directories
+#ifdef FFS_WIN
+    Zstring dbname = Zstring(Zstr("sync")) + (tempfile ? Zstr(".tmp") : Zstr("")) + SYNC_DB_FILE_ENDING;
+#elif defined FFS_LINUX
+    //files beginning with dots are hidden e.g. in Nautilus
+    Zstring dbname = Zstring(Zstr(".sync")) + (tempfile ? Zstr(".tmp") : Zstr("")) + SYNC_DB_FILE_ENDING;
+#endif
+
+    return baseMap.getBaseDirPf<side>() + dbname;
+}
+
+
+
+class FileInputStreamDB : public FileInputStream
+{
+public:
+    FileInputStreamDB(const Zstring& filename) : //throw FileError
+        FileInputStream(filename)
+    {
+        //read FreeFileSync file identifier
+        char formatDescr[sizeof(FILE_FORMAT_DESCR)] = {};
+        Read(formatDescr, sizeof(formatDescr)); //throw FileError
+
+        if (!std::equal(FILE_FORMAT_DESCR, FILE_FORMAT_DESCR + sizeof(FILE_FORMAT_DESCR), formatDescr))
+            throw FileError(_("Incompatible synchronization database format:") + " \n" + "\"" + filename + "\"");
+    }
+
+private:
+};
+
+
+class FileOutputStreamDB : public FileOutputStream
+{
+public:
+    FileOutputStreamDB(const Zstring& filename) : //throw FileError
+        FileOutputStream(filename)
+    {
+        //write FreeFileSync file identifier
+        Write(FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR)); //throw FileError
+    }
+
+private:
+};
+}
+//#######################################################################################################################################
+
+
+class ReadDirInfo : public zen::ReadInputStream
+{
+public:
+    ReadDirInfo(wxInputStream& stream, const wxString& errorObjName, DirInformation& dirInfo) : ReadInputStream(stream, errorObjName)
+    {
+        //|-------------------------------------------------------------------------------------
+        //| ensure 32/64 bit portability: use fixed size data types only e.g. boost::uint32_t |
+        //|-------------------------------------------------------------------------------------
+
+        //read filter settings -> currently not required, but persisting it doesn't hurt
+        dirInfo.filter = HardFilter::loadFilter(getStream());
+        check();
+
+        //start recursion
+        execute(dirInfo.baseDirContainer);
+    }
+
+private:
+    void execute(DirContainer& dirCont) const
+    {
+        while (readNumberC<bool>())
+            readSubFile(dirCont);
+
+        while (readNumberC<bool>())
+            readSubLink(dirCont);
+
+        while (readNumberC<bool>())
+            readSubDirectory(dirCont);
+    }
+
+    void readSubFile(DirContainer& dirCont) const
+    {
+        //attention: order of function argument evaluation is undefined! So do it one after the other...
+        const Zstring shortName = readStringC<Zstring>(); //file name
+
+        const std::int64_t  modTime  = readNumberC<std::int64_t>();
+        const std::uint64_t fileSize = readNumberC<std::uint64_t>();
+
+        //const util::FileID fileIdentifier(stream_);
+        //check();
+
+        dirCont.addSubFile(shortName,
+                           FileDescriptor(modTime, fileSize));
+    }
+
+
+    void readSubLink(DirContainer& dirCont) const
+    {
+        //attention: order of function argument evaluation is undefined! So do it one after the other...
+        const Zstring      shortName  = readStringC<Zstring>(); //file name
+        const std::int64_t modTime    = readNumberC<std::int64_t>();
+        const Zstring      targetPath = readStringC<Zstring>(); //file name
+        const LinkDescriptor::LinkType linkType = static_cast<LinkDescriptor::LinkType>(readNumberC<std::int32_t>());
+
+        dirCont.addSubLink(shortName,
+                           LinkDescriptor(modTime, targetPath, linkType));
+    }
+
+
+    void readSubDirectory(DirContainer& dirCont) const
+    {
+        const Zstring shortName = readStringC<Zstring>(); //directory name
+        DirContainer& subDir = dirCont.addSubDir(shortName);
+        execute(subDir); //recurse
+    }
+};
+
+namespace
+{
+typedef std::string UniqueId;
+typedef std::shared_ptr<std::vector<char> > MemoryStreamPtr;  //byte stream representing DirInformation
+typedef std::map<UniqueId, MemoryStreamPtr> StreamMapping;    //list of streams ordered by session UUID
+}
+
+class ReadFileStream : public zen::ReadInputStream
+{
+public:
+    ReadFileStream(wxInputStream& stream, const wxString& filename, StreamMapping& streamList) : ReadInputStream(stream, filename)
+    {
+        //|-------------------------------------------------------------------------------------
+        //| ensure 32/64 bit portability: used fixed size data types only e.g. boost::uint32_t |
+        //|-------------------------------------------------------------------------------------
+
+        std::int32_t version = readNumberC<std::int32_t>();
+
+        if (version != FILE_FORMAT_VER) //read file format version
+            throw FileError(_("Incompatible synchronization database format:") + " \n" + "\"" + filename.c_str() + "\"");
+
+        streamList.clear();
+
+        boost::uint32_t dbCount = readNumberC<boost::uint32_t>(); //number of databases: one for each sync-pair
+        while (dbCount-- != 0)
+        {
+            //DB id of partner databases
+            const CharArray tmp2 = readArrayC();
+            const std::string sessionID(tmp2->begin(), tmp2->end());
+
+            CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation)
+
+            streamList.insert(std::make_pair(sessionID, buffer));
+        }
+    }
+};
+
+namespace
+{
+StreamMapping loadStreams(const Zstring& filename) //throw FileError
+{
+    if (!zen::fileExists(filename))
+        throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + " \n\n" +
+                                           _("One of the FreeFileSync database files is not yet existing:") + " \n" +
+                                           "\"" + filename + "\"");
+
+    try
+    {
+        //read format description (uncompressed)
+        FileInputStreamDB uncompressed(filename); //throw FileError
+
+        wxZlibInputStream input(uncompressed, wxZLIB_ZLIB);
+
+        StreamMapping streamList;
+        ReadFileStream(input, toWx(filename), streamList);
+        return streamList;
+    }
+    catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file
+    {
+        throw FileError(_("Error reading from synchronization database:") + " (bad_alloc)");
+    }
+}
+
+
+DirInfoPtr parseStream(const std::vector<char>& stream, const Zstring& fileName) //throw FileError -> return value always bound!
+{
+    try
+    {
+        //read streams into DirInfo
+        auto dirInfo = std::make_shared<DirInformation>();
+        wxMemoryInputStream buffer(&stream[0], stream.size()); //convert char-array to inputstream: no copying, ownership not transferred
+        ReadDirInfo(buffer, toWx(fileName), *dirInfo); //throw FileError
+        return dirInfo;
+    }
+    catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file
+    {
+        throw FileError(_("Error reading from synchronization database:") + " (bad_alloc)");
+    }
+}
+}
+
+
+std::pair<DirInfoPtr, DirInfoPtr> zen::loadFromDisk(const BaseDirMapping& baseMapping) //throw FileError
+{
+    const Zstring fileNameLeft  = getDBFilename<LEFT_SIDE>(baseMapping);
+    const Zstring fileNameRight = getDBFilename<RIGHT_SIDE>(baseMapping);
+
+    //read file data: list of session ID + DirInfo-stream
+    const StreamMapping streamListLeft  = ::loadStreams(fileNameLeft);  //throw FileError
+    const StreamMapping streamListRight = ::loadStreams(fileNameRight); //throw FileError
+
+    //find associated session: there can be at most one session within intersection of left and right ids
+    StreamMapping::const_iterator streamLeft  = streamListLeft .end();
+    StreamMapping::const_iterator streamRight = streamListRight.end();
+    for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft)
+    {
+        auto iterRight = streamListRight.find(iterLeft->first);
+        if (iterRight != streamListRight.end())
+        {
+            streamLeft  = iterLeft;
+            streamRight = iterRight;
+            break;
+        }
+    }
+
+    if (streamLeft  == streamListLeft .end() ||
+        streamRight == streamListRight.end() ||
+        !streamLeft ->second.get() ||
+        !streamRight->second.get())
+        throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + " \n\n" +
+                                           _("Database files do not share a common synchronization session:") + " \n" +
+                                           "\"" + fileNameLeft  + "\"\n" +
+                                           "\"" + fileNameRight + "\"");
+    //read streams into DirInfo
+    DirInfoPtr dirInfoLeft  = parseStream(*streamLeft ->second, fileNameLeft);  //throw FileError
+    DirInfoPtr dirInfoRight = parseStream(*streamRight->second, fileNameRight); //throw FileError
+
+    return std::make_pair(dirInfoLeft, dirInfoRight);
+}
+
+
+//-------------------------------------------------------------------------------------------------------------------------
+template <SelectedSide side>
+class SaveDirInfo : public WriteOutputStream
+{
+public:
+    SaveDirInfo(const BaseDirMapping& baseMapping, const DirContainer* oldDirInfo, const wxString& errorObjName, wxOutputStream& stream) : WriteOutputStream(errorObjName, stream)
+    {
+        //save filter settings
+        baseMapping.getFilter()->saveFilter(getStream());
+        check();
+
+        //start recursion
+        execute(baseMapping, oldDirInfo);
+    }
+
+private:
+    void execute(const HierarchyObject& hierObj, const DirContainer* oldDirInfo)
+    {
+        std::for_each(hierObj.refSubFiles().begin(), hierObj.refSubFiles().end(), boost::bind(&SaveDirInfo::processFile, this, _1, oldDirInfo));
+        writeNumberC<bool>(false); //mark last entry
+        std::for_each(hierObj.refSubLinks().begin(), hierObj.refSubLinks().end(), boost::bind(&SaveDirInfo::processLink, this, _1, oldDirInfo));
+        writeNumberC<bool>(false); //mark last entry
+        std::for_each(hierObj.refSubDirs ().begin(), hierObj.refSubDirs ().end(), boost::bind(&SaveDirInfo::processDir,  this, _1, oldDirInfo));
+        writeNumberC<bool>(false); //mark last entry
+    }
+
+    void processFile(const FileMapping& fileMap, const DirContainer* oldParentDir)
+    {
+        if (fileMap.getCategory() == FILE_EQUAL) //data in sync: write current state
+        {
+            if (!fileMap.isEmpty<side>())
+            {
+                writeNumberC<bool>(true); //mark beginning of entry
+                writeStringC(fileMap.getShortName<side>()); //save respecting case! (Windows)
+                writeNumberC<std:: int64_t>(to<std:: int64_t>(fileMap.getLastWriteTime<side>())); //last modification time
+                writeNumberC<std::uint64_t>(to<std::uint64_t>(fileMap.getFileSize<side>())); //filesize
+            }
+        }
+        else //not in sync: reuse last synchronous state
+        {
+            if (oldParentDir) //no data is also a "synchronous state"!
+            {
+                auto iter = oldParentDir->files.find(fileMap.getObjShortName());
+                if (iter != oldParentDir->files.end())
+                {
+                    writeNumberC<bool>(true); //mark beginning of entry
+                    writeStringC(iter->first); //save respecting case! (Windows)
+                    writeNumberC<std:: int64_t>(to<std:: int64_t>(iter->second.lastWriteTimeRaw)); //last modification time
+                    writeNumberC<std::uint64_t>(to<std::uint64_t>(iter->second.fileSize)); //filesize
+                }
+            }
+        }
+    }
+
+    void processLink(const SymLinkMapping& linkObj, const DirContainer* oldParentDir)
+    {
+        if (linkObj.getLinkCategory() == SYMLINK_EQUAL) //data in sync: write current state
+        {
+            if (!linkObj.isEmpty<side>())
+            {
+                writeNumberC<bool>(true); //mark beginning of entry
+                writeStringC(linkObj.getShortName<side>()); //save respecting case! (Windows)
+                writeNumberC<std::int64_t>(to<std::int64_t>(linkObj.getLastWriteTime<side>())); //last modification time
+                writeStringC(linkObj.getTargetPath<side>());
+                writeNumberC<std::int32_t>(linkObj.getLinkType<side>());
+            }
+        }
+        else //not in sync: reuse last synchronous state
+        {
+            if (oldParentDir) //no data is also a "synchronous state"!
+            {
+                auto iter = oldParentDir->links.find(linkObj.getObjShortName());
+                if (iter != oldParentDir->links.end())
+                {
+                    writeNumberC<bool>(true); //mark beginning of entry
+                    writeStringC(iter->first); //save respecting case! (Windows)
+                    writeNumberC<std::int64_t>(to<std::int64_t>(iter->second.lastWriteTimeRaw)); //last modification time
+                    writeStringC(iter->second.targetPath);
+                    writeNumberC<std::int32_t>(iter->second.type);
+                }
+            }
+        }
+    }
+
+    void processDir(const DirMapping& dirMap, const DirContainer* oldParentDir)
+    {
+        const DirContainer* oldDir     = NULL;
+        const Zstring*      oldDirName = NULL;
+        if (oldParentDir) //no data is also a "synchronous state"!
+        {
+            auto iter = oldParentDir->dirs.find(dirMap.getObjShortName());
+            if (iter != oldParentDir->dirs.end())
+            {
+                oldDirName = &iter->first;
+                oldDir     = &iter->second;
+            }
+        }
+
+        CompareDirResult cat = dirMap.getDirCategory();
+
+        if (cat == DIR_EQUAL) //data in sync: write current state
+        {
+            if (!dirMap.isEmpty<side>())
+            {
+                writeNumberC<bool>(true); //mark beginning of entry
+                writeStringC(dirMap.getShortName<side>()); //save respecting case! (Windows)
+                execute(dirMap, oldDir); //recurse
+            }
+        }
+        else //not in sync: reuse last synchronous state
+        {
+            if (oldDir)
+            {
+                writeNumberC<bool>(true);  //mark beginning of entry
+                writeStringC(*oldDirName); //save respecting case! (Windows)
+                execute(dirMap, oldDir); //recurse
+                return;
+            }
+            //no data is also a "synchronous state"!
+
+            //else: not in sync AND no "last synchronous state"
+            //we cannot simply skip the whole directory, since sub-items might be in sync
+            //Example: directories on left and right differ in case while sub-files are equal
+            switch (cat)
+            {
+                case DIR_LEFT_SIDE_ONLY: //sub-items cannot be in sync
+                    break;
+                case DIR_RIGHT_SIDE_ONLY: //sub-items cannot be in sync
+                    break;
+                case DIR_EQUAL:
+                    assert(false);
+                    break;
+                case DIR_DIFFERENT_METADATA:
+                    writeNumberC<bool>(true);
+                    writeStringC(dirMap.getShortName<side>());
+                    //ATTENTION: strictly this is a violation of the principle of reporting last synchronous state!
+                    //however in this case this will result in "last sync unsuccessful" for this directory within <automatic> algorithm, which is fine
+                    execute(dirMap, oldDir); //recurse and save sub-items which are in sync
+                    break;
+            }
+        }
+    }
+};
+
+
+class WriteFileStream : public WriteOutputStream
+{
+public:
+    WriteFileStream(const StreamMapping& streamList, const wxString& filename, wxOutputStream& stream) : WriteOutputStream(filename, stream)
+    {
+        //save file format version
+        writeNumberC<std::int32_t>(FILE_FORMAT_VER);
+
+        writeNumberC<boost::uint32_t>(static_cast<boost::uint32_t>(streamList.size())); //number of database records: one for each sync-pair
+
+        for (StreamMapping::const_iterator i = streamList.begin(); i != streamList.end(); ++i)
+        {
+            //sync session id
+            writeArrayC(std::vector<char>(i->first.begin(), i->first.end()));
+
+            //write DirInformation stream
+            writeArrayC(*(i->second));
+        }
+    }
+};
+
+
+//save/load DirContainer
+void saveFile(const StreamMapping& streamList, const Zstring& filename) //throw FileError
+{
+    {
+        //write format description (uncompressed)
+        FileOutputStreamDB uncompressed(filename); //throw FileError
+
+        wxZlibOutputStream output(uncompressed, 4, wxZLIB_ZLIB);
+        /* 4 - best compromise between speed and compression: (scanning 200.000 objects)
+        0 (uncompressed)        8,95 MB -  422 ms
+        2                       2,07 MB -  470 ms
+        4                       1,87 MB -  500 ms
+        6                       1,77 MB -  613 ms
+        9 (maximal compression) 1,74 MB - 3330 ms */
+
+        WriteFileStream(streamList, toWx(filename), output);
+    }
+    //(try to) hide database file
+#ifdef FFS_WIN
+    ::SetFileAttributes(zen::applyLongPathPrefix(filename).c_str(), FILE_ATTRIBUTE_HIDDEN);
+#endif
+}
+
+
+bool equalEntry(const MemoryStreamPtr& lhs, const MemoryStreamPtr& rhs)
+{
+    if (!lhs.get() || !rhs.get())
+        return lhs.get() == rhs.get();
+
+    return *lhs == *rhs;
+}
+
+
+void zen::saveToDisk(const BaseDirMapping& baseMapping) //throw FileError
+{
+    //transactional behaviour! write to tmp files first
+    const Zstring dbNameLeftTmp  = getDBFilename<LEFT_SIDE >(baseMapping, true);
+    const Zstring dbNameRightTmp = getDBFilename<RIGHT_SIDE>(baseMapping, true);
+
+    const Zstring dbNameLeft  = getDBFilename<LEFT_SIDE >(baseMapping);
+    const Zstring dbNameRight = getDBFilename<RIGHT_SIDE>(baseMapping);
+
+    //delete old tmp file, if necessary -> throws if deletion fails!
+    removeFile(dbNameLeftTmp);  //
+    removeFile(dbNameRightTmp); //throw FileError
+
+    //(try to) load old database files...
+    StreamMapping streamListLeft;
+    StreamMapping streamListRight;
+
+    try //read file data: list of session ID + DirInfo-stream
+    {
+        streamListLeft = ::loadStreams(dbNameLeft);
+    }
+    catch (FileError&) {} //if error occurs: just overwrite old file! User is already informed about issues right after comparing!
+    try
+    {
+        streamListRight = ::loadStreams(dbNameRight);
+    }
+    catch (FileError&) {}
+
+    //find associated session: there can be at most one session within intersection of left and right ids
+    StreamMapping::iterator streamLeft  = streamListLeft .end();
+    StreamMapping::iterator streamRight = streamListRight.end();
+    for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft)
+    {
+        auto iterRight = streamListRight.find(iterLeft->first);
+        if (iterRight != streamListRight.end())
+        {
+            streamLeft  = iterLeft;
+            streamRight = iterRight;
+            break;
+        }
+    }
+
+    //(try to) read old DirInfo
+    DirInfoPtr oldDirInfoLeft;
+    DirInfoPtr oldDirInfoRight;
+    try
+    {
+        if (streamLeft  != streamListLeft .end() &&
+            streamRight != streamListRight.end() &&
+            streamLeft ->second.get() &&
+            streamRight->second.get())
+        {
+            oldDirInfoLeft  = parseStream(*streamLeft ->second, dbNameLeft); //throw FileError
+            oldDirInfoRight = parseStream(*streamRight->second, dbNameRight); //throw FileError
+        }
+    }
+    catch (FileError&)
+    {
+        //if error occurs: just overwrite old file! User is already informed about issues right after comparing!
+        oldDirInfoLeft .reset(); //read both or none!
+        oldDirInfoRight.reset(); //
+    }
+
+    //create new database entries
+    MemoryStreamPtr newStreamLeft = std::make_shared<std::vector<char>>();
+    {
+        wxMemoryOutputStream buffer;
+        const DirContainer* oldDir = oldDirInfoLeft.get() ? &oldDirInfoLeft->baseDirContainer : NULL;
+        SaveDirInfo<LEFT_SIDE>(baseMapping, oldDir, toWx(dbNameLeft), buffer);
+        newStreamLeft->resize(buffer.GetSize());               //convert output stream to char-array
+        buffer.CopyTo(&(*newStreamLeft)[0], buffer.GetSize()); //
+    }
+
+    MemoryStreamPtr newStreamRight = std::make_shared<std::vector<char>>();
+    {
+        wxMemoryOutputStream buffer;
+        const DirContainer* oldDir = oldDirInfoRight.get() ? &oldDirInfoRight->baseDirContainer : NULL;
+        SaveDirInfo<RIGHT_SIDE>(baseMapping, oldDir, toWx(dbNameRight), buffer);
+        newStreamRight->resize(buffer.GetSize());               //convert output stream to char-array
+        buffer.CopyTo(&(*newStreamRight)[0], buffer.GetSize()); //
+    }
+
+    //check if there is some work to do at all
+    {
+        const bool updateRequiredLeft  = streamLeft  == streamListLeft .end() || !equalEntry(newStreamLeft,  streamLeft ->second);
+        const bool updateRequiredRight = streamRight == streamListRight.end() || !equalEntry(newStreamRight, streamRight->second);
+        //some users monitor the *.ffs_db file with RTS => don't touch the file if it isnt't strictly needed
+        if (!updateRequiredLeft && !updateRequiredRight)
+            return;
+    }
+
+    //create/update DirInfo-streams
+    std::string sessionID = zen::generateGUID();
+
+    //erase old session data
+    if (streamLeft != streamListLeft.end())
+        streamListLeft.erase(streamLeft);
+    if (streamRight != streamListRight.end())
+        streamListRight.erase(streamRight);
+
+    //fill in new
+    streamListLeft .insert(std::make_pair(sessionID, newStreamLeft));
+    streamListRight.insert(std::make_pair(sessionID, newStreamRight));
+
+    //write (temp-) files...
+    zen::ScopeGuard guardTempFileLeft = zen::makeGuard([&]() {zen::removeFile(dbNameLeftTmp); });
+    saveFile(streamListLeft, dbNameLeftTmp);  //throw FileError
+
+    zen::ScopeGuard guardTempFileRight = zen::makeGuard([&]() {zen::removeFile(dbNameRightTmp); });
+    saveFile(streamListRight, dbNameRightTmp); //throw FileError
+
+    //operation finished: rename temp files -> this should work transactionally:
+    //if there were no write access, creation of temp files would have failed
+    removeFile(dbNameLeft);
+    removeFile(dbNameRight);
+    renameFile(dbNameLeftTmp,  dbNameLeft);  //throw FileError;
+    renameFile(dbNameRightTmp, dbNameRight); //throw FileError;
+
+    guardTempFileLeft. dismiss(); //no need to delete temp file anymore
+    guardTempFileRight.dismiss(); //
+}