3.1

1 files changed, 358 insertions, 147 deletions

diff --git a/fileHierarchy.cpp b/fileHierarchy.cpp
index 4c6c9a4b..f0458b8e 100644
--- a/fileHierarchy.cpp
+++ b/fileHierarchy.cpp
@@ -7,6 +7,8 @@
 #include <wx/intl.h>
 #include "shared/stringConv.h"
 #include "shared/fileHandling.h"
+#include <wx/mstream.h>
+
 #ifdef FFS_WIN
 #include <wx/msw/wrapwin.h> //includes "windows.h"
 #endif
@@ -193,7 +195,7 @@ const Zstring& FreeFileSync::getSyncDBFilename()
 inline
 Zstring readString(wxInputStream& stream)  //read string from file stream
 {
-    const unsigned int strLength = readNumber<unsigned int>(stream);
+    const size_t strLength = readNumber<size_t>(stream);
     if (strLength <= 1000)
     {
         DefaultChar buffer[1000];
@@ -212,161 +214,72 @@ Zstring readString(wxInputStream& stream)  //read string from file stream
 inline
 void writeString(wxOutputStream& stream, const Zstring& str)  //write string to filestream
 {
-    globalFunctions::writeNumber<unsigned int>(stream, str.length());
+    globalFunctions::writeNumber<size_t>(stream, str.length());
     stream.Write(str.c_str(), sizeof(DefaultChar) * str.length());
 }
 
 
 //-------------------------------------------------------------------------------------------------------------------------------
 const char FILE_FORMAT_DESCR[] = "FreeFileSync";
-const int FILE_FORMAT_VER = 1;
+const int FILE_FORMAT_VER = 2;
 //-------------------------------------------------------------------------------------------------------------------------------
 
-
-template <SelectedSide side>
-struct IsNonEmpty
-{
-    bool operator()(const FileSystemObject& fsObj) const
-    {
-        return !fsObj.isEmpty<side>();
-    }
-};
-
-
-template <SelectedSide side>
-class SaveRecursively
+class ReadInputStream
 {
-public:
-    SaveRecursively(const BaseDirMapping& baseMapping, const Zstring& filename, wxOutputStream& stream) : filename_(filename), stream_(stream)
-    {
-        //save file format version
-        writeNumberC<int>(FILE_FORMAT_VER);
-
-        //save filter settings
-        writeNumberC<bool>(baseMapping.getFilter().filterActive);
-        writeStringC(baseMapping.getFilter().includeFilter.c_str());
-        writeStringC(baseMapping.getFilter().excludeFilter.c_str());
-
-        //start recursion
-        execute(baseMapping);
-    }
-
-private:
-    template<typename Iterator, typename Function>
-    friend Function std::for_each(Iterator, Iterator, Function);
+protected:
+    ReadInputStream(wxInputStream& stream, const Zstring& errorObjName) : stream_(stream), errorObjName_(errorObjName) {}
 
-    void execute(const HierarchyObject& hierObj)
-    {
-        writeNumberC<unsigned int>(std::count_if(hierObj.subFiles.begin(), hierObj.subFiles.end(), IsNonEmpty<side>())); //number of (existing) files
-        std::for_each(hierObj.subFiles.begin(), hierObj.subFiles.end(), *this);
-
-        writeNumberC<unsigned int>(std::count_if(hierObj.subDirs.begin(), hierObj.subDirs.end(), IsNonEmpty<side>())); //number of (existing) directories
-        std::for_each(hierObj.subDirs.begin(), hierObj.subDirs.end(), *this);
-    }
-
-    void operator()(const FileMapping& fileMap)
-    {
-        if (!fileMap.isEmpty<side>())
-        {
-            writeStringC(fileMap.getObjShortName()); //file name
-            writeNumberC<long>(         fileMap.getLastWriteTime<side>().GetHi()); //last modification time
-            writeNumberC<unsigned long>(fileMap.getLastWriteTime<side>().GetLo()); //
-            writeNumberC<unsigned long>(fileMap.getFileSize<side>().GetHi()); //filesize
-            writeNumberC<unsigned long>(fileMap.getFileSize<side>().GetLo()); //
-        }
-    }
-
-    void operator()(const DirMapping& dirMap)
+    void check()
     {
-        if (!dirMap.isEmpty<side>())
-        {
-            writeStringC(dirMap.getObjShortName()); //directory name
-            execute(dirMap); //recurse
-        }
+        if (stream_.GetLastError() != wxSTREAM_NO_ERROR)
+            throw FileError(wxString(_("Error reading from synchronization database:")) + wxT(" \n") +
+                            wxT("\"") +  zToWx(errorObjName_) + wxT("\""));
     }
 
     template <class T>
-    void writeNumberC(T number) //checked write operation
+    T readNumberC() //checked read operation
     {
-        writeNumber<T>(stream_, number);
+        T output = readNumber<T>(stream_);
         check();
+        return output;
     }
 
-    void writeStringC(const Zstring& str) //checked write operation
+    Zstring readStringC() //checked read operation
     {
-        writeString(stream_, str);
+        Zstring output = readString(stream_);
         check();
+        return output;
     }
 
-    void check()
-    {
-        if (stream_.GetLastError() != wxSTREAM_NO_ERROR)
-            throw FileError(wxString(_("Error writing to synchronization database:")) + wxT(" \n") +
-                            wxT("\"") + zToWx(filename_) + wxT("\""));
-    }
-
-    const Zstring& filename_; //used for error text only
-    wxOutputStream& stream_;
-};
-
-
-//save/load DirContainer
-void FreeFileSync::saveToDisk(const BaseDirMapping& baseMapping, SelectedSide side, const Zstring& filename) //throw (FileError)
-{
-    try //(try to) delete old file: overwriting directly doesn't always work
-    {
-        removeFile(filename, false);
-    }
-    catch (...) {}
-
-    try
-    {
-        //write format description (uncompressed)
-        wxFFileOutputStream uncompressed(zToWx(filename), wxT("wb"));
-        uncompressed.Write(FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR));
-        if (uncompressed.GetLastError() != wxSTREAM_NO_ERROR)
-            throw FileError(wxString(_("Error writing to synchronization database:")) + wxT(" \n") +
-                            wxT("\"") + zToWx(filename) + wxT("\""));
-
-        wxZlibOutputStream output(uncompressed, 4, wxZLIB_ZLIB);
-        /* 4 - best compromise between speed and compression: (scanning 200.000 objects)
-        0 (uncompressed)        8,95 MB -  422 ms
-        2                       2,07 MB -  470 ms
-        4                       1,87 MB -  500 ms
-        6                       1,77 MB -  613 ms
-        9 (maximal compression) 1,74 MB - 3330 ms */
-
-        if (side == LEFT_SIDE)
-            SaveRecursively<LEFT_SIDE>(baseMapping, filename, output);
-        else
-            SaveRecursively<RIGHT_SIDE>(baseMapping, filename, output);
-    }
-    catch (FileError&)
+    typedef boost::shared_ptr<std::vector<char> > CharArray;
+    CharArray readArrayC()
     {
-        try //(try to) delete erroneous file
+        CharArray buffer(new std::vector<char>);
+        const size_t byteCount = readNumberC<size_t>();
+        if (byteCount > 0)
         {
-            removeFile(filename, false);
+            buffer->resize(byteCount);
+            stream_.Read(&(*buffer)[0], byteCount);
+            check();
+            if (stream_.LastRead() != byteCount) //some additional check
+                throw FileError(wxString(_("Error reading from synchronization database:")) + wxT(" \n") +
+                                wxT("\"") +  zToWx(errorObjName_) + wxT("\""));
         }
-        catch (...) {}
-        throw;
+        return buffer;
     }
 
-    //(try to) hide database file
-#ifdef FFS_WIN
-    ::SetFileAttributes(filename.c_str(), FILE_ATTRIBUTE_HIDDEN);
-#endif
-}
+protected:
+    wxInputStream& stream_;
+private:
+    const Zstring& errorObjName_; //used for error text only
+};
 
 
-//-------------------------------------------------------------------------------------------------------------------------
-class ReadRecursively
+class ReadDirInfo : public ReadInputStream
 {
 public:
-    ReadRecursively(wxInputStream& stream, const Zstring& filename, DirInformation& dirInfo) : filename_(filename), stream_(stream)
+    ReadDirInfo(wxInputStream& stream, const Zstring& errorObjName, DirInformation& dirInfo) : ReadInputStream(stream, errorObjName)
     {
-        if (readNumberC<int>() != FILE_FORMAT_VER) //read file format version
-            throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + zToWx(filename_) + wxT("\""));
-
         //save filter settings
         dirInfo.filterActive  = readNumberC<bool>();
         dirInfo.includeFilter = readStringC();
@@ -409,36 +322,58 @@ private:
         DirContainer& subDir = dirCont.addSubDir(shortName);
         execute(subDir); //recurse
     }
+};
 
-    void check()
-    {
-        if (stream_.GetLastError() != wxSTREAM_NO_ERROR)
-            throw FileError(wxString(_("Error reading from synchronization database:")) + wxT(" \n") +
-                            wxT("\"") +  zToWx(filename_) + wxT("\""));
-    }
 
-    template <class T>
-    T readNumberC() //checked read operation
+typedef boost::shared_ptr<std::vector<char> > MemoryStreamPtr;     //byte stream representing DirInformation
+typedef std::map<Utility::UniqueId, MemoryStreamPtr> DirectoryTOC; //list of streams ordered by a UUID pointing to their partner database
+typedef std::pair<Utility::UniqueId, DirectoryTOC> DbStreamData;   //header data: UUID representing this database, item data: list of dir-streams
+/* Example
+left side              right side
+---------              ----------
+DB-ID 123     <-\  /-> DB-ID 567
+                 \/
+Partner-ID 111   /\    Partner-ID 222
+Partner-ID 567 -/  \-  Partner-ID 123
+    ...                     ...
+*/
+
+class ReadFileStream : public ReadInputStream
+{
+public:
+    ReadFileStream(wxInputStream& stream, const Zstring& filename, DbStreamData& output) : ReadInputStream(stream, filename)
     {
-        T output = readNumber<T>(stream_);
-        check();
-        return output;
-    }
+        if (readNumberC<int>() != FILE_FORMAT_VER) //read file format version
+            throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + zToWx(filename) + wxT("\""));
 
-    Zstring readStringC() //checked read operation
-    {
-        Zstring output = readString(stream_);
+        //read DB id
+        output.first = Utility::UniqueId(stream_);
         check();
-        return output;
-    }
 
-    const Zstring& filename_; //used for error text only
-    wxInputStream& stream_;
+        DirectoryTOC& dbList = output.second;
+        dbList.clear();
+
+        size_t dbCount = readNumberC<size_t>(); //number of databases: one for each sync-pair
+        while (dbCount-- != 0)
+        {
+            const Utility::UniqueId partnerID(stream_); //DB id of partner databases
+            check();
+
+            CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation)
+
+            dbList.insert(std::make_pair(partnerID, buffer));
+        }
+    }
 };
 
 
-boost::shared_ptr<const DirInformation> FreeFileSync::loadFromDisk(const Zstring& filename) //throw (FileError)
+DbStreamData loadFile(const Zstring& filename) //throw (FileError)
 {
+    if (!FreeFileSync::fileExists(filename))
+        throw FileError(wxString(_("Initial synchronization.")) + wxT(" \n") +
+                        wxT("(") + _("No database file existing yet:") + wxT(" \"") + zToWx(filename) + wxT("\")"));
+
+
     //read format description (uncompressed)
     wxFFileInputStream uncompressed(zToWx(filename), wxT("rb"));
 
@@ -453,8 +388,284 @@ boost::shared_ptr<const DirInformation> FreeFileSync::loadFromDisk(const Zstring
 
     wxZlibInputStream input(uncompressed, wxZLIB_ZLIB);
 
-    boost::shared_ptr<DirInformation> dirInfo(new DirInformation);
-    ReadRecursively(input, filename, *dirInfo);  //read file/dir information
+    DbStreamData output;
+    ReadFileStream(input, filename, output);
+    return output;
+}
+
+
+std::pair<DirInfoPtr, DirInfoPtr> FreeFileSync::loadFromDisk(const BaseDirMapping& baseMapping) //throw (FileError)
+{
+    const Zstring fileNameLeft  = baseMapping.getDBFilename<LEFT_SIDE>();
+    const Zstring fileNameRight = baseMapping.getDBFilename<RIGHT_SIDE>();
+
+    //read file data: db ID + mapping of partner-ID/DirInfo-stream
+    const DbStreamData dbEntriesLeft  = ::loadFile(fileNameLeft);
+    const DbStreamData dbEntriesRight = ::loadFile(fileNameRight);
+
+    //find associated DirInfo-streams
+    DirectoryTOC::const_iterator dbLeft = dbEntriesLeft.second.find(dbEntriesRight.first); //find left db-entry that corresponds to right database
+    if (dbLeft == dbEntriesLeft.second.end())
+        throw FileError(wxString(_("Initial synchronization.")) + wxT(" \n") +
+                        wxT("(") + _("No database entry existing in file:") + wxT(" \"") + zToWx(fileNameLeft) + wxT("\")"));
+
+    DirectoryTOC::const_iterator dbRight = dbEntriesRight.second.find(dbEntriesLeft.first); //find left db-entry that corresponds to right database
+    if (dbRight == dbEntriesRight.second.end())
+        throw FileError(wxString(_("Initial synchronization.")) + wxT(" \n") +
+                        wxT("(") + _("No database entry existing in file:") + wxT(" \"") + zToWx(fileNameRight) + wxT("\")"));
+
+    //read streams into DirInfo
+    boost::shared_ptr<DirInformation> dirInfoLeft(new DirInformation);
+    wxMemoryInputStream buffer(&(*dbLeft->second)[0], dbLeft->second->size()); //convert char-array to inputstream: no copying, ownership not transferred
+    ReadDirInfo(buffer, fileNameLeft, *dirInfoLeft);  //read file/dir information
+
+    boost::shared_ptr<DirInformation> dirInfoRight(new DirInformation);
+    wxMemoryInputStream buffer2(&(*dbRight->second)[0], dbRight->second->size()); //convert char-array to inputstream: no copying, ownership not transferred
+    ReadDirInfo(buffer2, fileNameRight, *dirInfoRight);  //read file/dir information
+
+    return std::make_pair(dirInfoLeft, dirInfoRight);
+}
+
+
+//-------------------------------------------------------------------------------------------------------------------------
+
+template <SelectedSide side>
+struct IsNonEmpty
+{
+    bool operator()(const FileSystemObject& fsObj) const
+    {
+        return !fsObj.isEmpty<side>();
+    }
+};
+
+
+class WriteOutputStream
+{
+protected:
+    WriteOutputStream(const Zstring& errorObjName, wxOutputStream& stream) : stream_(stream), errorObjName_(errorObjName) {}
+
+    void check()
+    {
+        if (stream_.GetLastError() != wxSTREAM_NO_ERROR)
+            throw FileError(wxString(_("Error writing to synchronization database:")) + wxT(" \n") +
+                            wxT("\"") + zToWx(errorObjName_) + wxT("\""));
+    }
+
+    template <class T>
+    void writeNumberC(T number) //checked write operation
+    {
+        writeNumber<T>(stream_, number);
+        check();
+    }
+
+    void writeStringC(const Zstring& str) //checked write operation
+    {
+        writeString(stream_, str);
+        check();
+    }
+
+    void writeArrayC(const std::vector<char>& buffer)
+    {
+        writeNumberC<size_t>(buffer.size());
+        if (buffer.size() > 0)
+        {
+            stream_.Write(&buffer[0], buffer.size());
+            check();
+            if (stream_.LastWrite() != buffer.size()) //some additional check
+                throw FileError(wxString(_("Error writing to synchronization database:")) + wxT(" \n") +
+                                wxT("\"") +  zToWx(errorObjName_) + wxT("\""));
+        }
+    }
+
+protected:
+    wxOutputStream& stream_;
+private:
+    const Zstring& errorObjName_; //used for error text only!
+};
+
+
+template <SelectedSide side>
+class SaveDirInfo : public WriteOutputStream
+{
+public:
+    SaveDirInfo(const BaseDirMapping& baseMapping, const Zstring& errorObjName, wxOutputStream& stream) : WriteOutputStream(errorObjName, stream)
+    {
+        //save filter settings
+        writeNumberC<bool>(baseMapping.getFilter().filterActive);
+        writeStringC(baseMapping.getFilter().includeFilter.c_str());
+        writeStringC(baseMapping.getFilter().excludeFilter.c_str());
+
+        //start recursion
+        execute(baseMapping);
+    }
+
+private:
+    template<typename Iterator, typename Function>
+    friend Function std::for_each(Iterator, Iterator, Function);
+
+    void execute(const HierarchyObject& hierObj)
+    {
+        writeNumberC<unsigned int>(std::count_if(hierObj.subFiles.begin(), hierObj.subFiles.end(), IsNonEmpty<side>())); //number of (existing) files
+        std::for_each(hierObj.subFiles.begin(), hierObj.subFiles.end(), *this);
+
+        writeNumberC<unsigned int>(std::count_if(hierObj.subDirs.begin(), hierObj.subDirs.end(), IsNonEmpty<side>())); //number of (existing) directories
+        std::for_each(hierObj.subDirs.begin(), hierObj.subDirs.end(), *this);
+    }
+
+    void operator()(const FileMapping& fileMap)
+    {
+        if (!fileMap.isEmpty<side>())
+        {
+            writeStringC(fileMap.getObjShortName()); //file name
+            writeNumberC<long>(         fileMap.getLastWriteTime<side>().GetHi()); //last modification time
+            writeNumberC<unsigned long>(fileMap.getLastWriteTime<side>().GetLo()); //
+            writeNumberC<unsigned long>(fileMap.getFileSize<side>().GetHi()); //filesize
+            writeNumberC<unsigned long>(fileMap.getFileSize<side>().GetLo()); //
+        }
+    }
+
+    void operator()(const DirMapping& dirMap)
+    {
+        if (!dirMap.isEmpty<side>())
+        {
+            writeStringC(dirMap.getObjShortName()); //directory name
+            execute(dirMap); //recurse
+        }
+    }
+};
+
 
-    return dirInfo;
+class WriteFileStream : public WriteOutputStream
+{
+public:
+    WriteFileStream(const DbStreamData& input, const Zstring& filename, wxOutputStream& stream) : WriteOutputStream(filename, stream)
+    {
+        //save file format version
+        writeNumberC<int>(FILE_FORMAT_VER);
+
+        //write DB id
+        input.first.toStream(stream_);
+        check();
+
+        const DirectoryTOC& dbList = input.second;
+
+        writeNumberC<size_t>(dbList.size()); //number of database records: one for each sync-pair
+
+        for (DirectoryTOC::const_iterator i = dbList.begin(); i != dbList.end(); ++i)
+        {
+            i->first.toStream(stream_); //DB id of partner database
+            check();
+
+            writeArrayC(*(i->second)); //write DirInformation stream
+        }
+    }
+};
+
+
+//save/load DirContainer
+void saveFile(const DbStreamData& dbStream, const Zstring& filename) //throw (FileError)
+{
+    //write format description (uncompressed)
+    wxFFileOutputStream uncompressed(zToWx(filename), wxT("wb"));
+    uncompressed.Write(FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR));
+    if (uncompressed.GetLastError() != wxSTREAM_NO_ERROR)
+        throw FileError(wxString(_("Error writing to synchronization database:")) + wxT(" \n") +
+                        wxT("\"") + zToWx(filename) + wxT("\""));
+
+    wxZlibOutputStream output(uncompressed, 4, wxZLIB_ZLIB);
+    /* 4 - best compromise between speed and compression: (scanning 200.000 objects)
+    0 (uncompressed)        8,95 MB -  422 ms
+    2                       2,07 MB -  470 ms
+    4                       1,87 MB -  500 ms
+    6                       1,77 MB -  613 ms
+    9 (maximal compression) 1,74 MB - 3330 ms */
+
+    WriteFileStream(dbStream, filename, output);
+
+    //(try to) hide database file
+#ifdef FFS_WIN
+    output.Close();
+    ::SetFileAttributes(filename.c_str(), FILE_ATTRIBUTE_HIDDEN);
+#endif
 }
+
+
+void FreeFileSync::saveToDisk(const BaseDirMapping& baseMapping) //throw (FileError)
+{
+    //transactional behaviour! write to tmp files first
+    const Zstring fileNameLeftTmp  = baseMapping.getDBFilename<LEFT_SIDE>()  + DefaultStr(".tmp");
+    const Zstring fileNameRightTmp = baseMapping.getDBFilename<RIGHT_SIDE>() + DefaultStr(".tmp");;
+
+    //delete old tmp file, if necessary -> throws if deletion fails!
+    removeFile(fileNameLeftTmp,  false);
+    removeFile(fileNameRightTmp, false);
+
+    try
+    {
+        //load old database files...
+
+        //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw!
+        DbStreamData dbEntriesLeft;
+        if (FreeFileSync::fileExists(baseMapping.getDBFilename<LEFT_SIDE>()))
+            try
+            {
+                dbEntriesLeft = ::loadFile(baseMapping.getDBFilename<LEFT_SIDE>());
+            }
+            catch(FileError&) {} //if error occurs: just overwrite old file! User is informed about issues right after comparing!
+        //else -> dbEntriesLeft has empty mapping, but already a DB-ID!
+
+        //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw!
+        DbStreamData dbEntriesRight;
+        if (FreeFileSync::fileExists(baseMapping.getDBFilename<RIGHT_SIDE>()))
+            try
+            {
+                dbEntriesRight = ::loadFile(baseMapping.getDBFilename<RIGHT_SIDE>());
+            }
+            catch(FileError&) {} //if error occurs: just overwrite old file! User is informed about issues right after comparing!
+
+        //create new database entries
+        MemoryStreamPtr dbEntryLeft(new std::vector<char>);
+        {
+            wxMemoryOutputStream buffer;
+            SaveDirInfo<LEFT_SIDE>(baseMapping, baseMapping.getDBFilename<LEFT_SIDE>(), buffer);
+            dbEntryLeft->resize(buffer.GetSize());               //convert output stream to char-array
+            buffer.CopyTo(&(*dbEntryLeft)[0], buffer.GetSize()); //
+        }
+
+        MemoryStreamPtr dbEntryRight(new std::vector<char>);
+        {
+            wxMemoryOutputStream buffer;
+            SaveDirInfo<RIGHT_SIDE>(baseMapping, baseMapping.getDBFilename<RIGHT_SIDE>(), buffer);
+            dbEntryRight->resize(buffer.GetSize());               //convert output stream to char-array
+            buffer.CopyTo(&(*dbEntryRight)[0], buffer.GetSize()); //
+        }
+
+        //create/update DirInfo-streams
+        dbEntriesLeft.second[dbEntriesRight.first] = dbEntryLeft;
+        dbEntriesRight.second[dbEntriesLeft.first] = dbEntryRight;
+
+        //write (temp-) files...
+        saveFile(dbEntriesLeft,  fileNameLeftTmp); //throw (FileError)
+        saveFile(dbEntriesRight, fileNameRightTmp); //throw (FileError)
+
+        //operation finished: rename temp files -> this should work transactionally:
+        //if there were no write access, creation of temp files would have failed
+        removeFile(baseMapping.getDBFilename<LEFT_SIDE>(), false);
+        removeFile(baseMapping.getDBFilename<RIGHT_SIDE>(), false);
+        renameFile(fileNameLeftTmp, baseMapping.getDBFilename<LEFT_SIDE>()); //throw (FileError);
+        renameFile(fileNameRightTmp, baseMapping.getDBFilename<RIGHT_SIDE>()); //throw (FileError);
+
+    }
+    catch (...)
+    {
+        try //clean up: (try to) delete old tmp file
+        {
+            removeFile(fileNameLeftTmp, false);
+            removeFile(fileNameRightTmp, false);
+        }
+        catch (...) {}
+
+        throw;
+    }
+}
+