From b338e29fd3eaf700f8c8360aa0310048ba941d54 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Fri, 18 Apr 2014 17:12:46 +0200 Subject: 3.19 --- library/db_file.cpp | 386 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 223 insertions(+), 163 deletions(-) (limited to 'library/db_file.cpp') diff --git a/library/db_file.cpp b/library/db_file.cpp index 1d48dbdf..ec1c4464 100644 --- a/library/db_file.cpp +++ b/library/db_file.cpp @@ -30,10 +30,29 @@ namespace { //------------------------------------------------------------------------------------------------------------------------------- const char FILE_FORMAT_DESCR[] = "FreeFileSync"; -const int FILE_FORMAT_VER = 6; +const int FILE_FORMAT_VER = 7; //------------------------------------------------------------------------------------------------------------------------------- +template inline +Zstring getDBFilename(const BaseDirMapping& baseMap, bool tempfile = false) +{ + //Linux and Windows builds are binary incompatible: char/wchar_t case, sensitive/insensitive + //32 and 64 bit db files ARE designed to be binary compatible! + //Give db files different names. + //make sure they end with ".ffs_db". These files will not be included into comparison when located in base sync directories +#ifdef FFS_WIN + Zstring dbname = Zstring(Zstr("sync")) + (tempfile ? Zstr(".tmp") : Zstr("")) + SYNC_DB_FILE_ENDING; +#elif defined FFS_LINUX + //files beginning with dots are hidden e.g. in Nautilus + Zstring dbname = Zstring(Zstr(".sync")) + (tempfile ? Zstr(".tmp") : Zstr("")) + SYNC_DB_FILE_ENDING; +#endif + + return baseMap.getBaseDir() + dbname; +} + + + class FileInputStreamDB : public FileInputStream { public: @@ -45,7 +64,7 @@ public: Read(formatDescr, sizeof(formatDescr)); //throw (FileError) if (!std::equal(FILE_FORMAT_DESCR, FILE_FORMAT_DESCR + sizeof(FILE_FORMAT_DESCR), formatDescr)) - throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + zToWx(filename) + wxT("\"")); + throw FileError(_("Incompatible synchronization database format:") + " \n" + "\"" + filename + "\""); } private: @@ -138,115 +157,162 @@ private: namespace { typedef std::string UniqueId; -typedef std::shared_ptr > MemoryStreamPtr; //byte stream representing DirInformation -typedef std::map DirectoryTOC; //list of streams ordered by a UUID pointing to their partner database -typedef std::pair DbStreamData; //header data: UUID representing this database, item data: list of dir-streams +typedef std::shared_ptr > MemoryStreamPtr; //byte stream representing DirInformation +typedef std::map StreamMapping; //list of streams ordered by session UUID } -/* Example -left side right side ---------- ---------- -DB-ID 123 <-\ /-> DB-ID 567 - \/ -Partner-ID 111 /\ Partner-ID 222 -Partner-ID 567 _/ \_ Partner-ID 123 - ... ... -*/ class ReadFileStream : public zen::ReadInputStream { public: - ReadFileStream(wxInputStream& stream, const wxString& filename, DbStreamData& output, int& versionId) : ReadInputStream(stream, filename) + ReadFileStream(wxInputStream& stream, const wxString& filename, StreamMapping& streamList, bool leftSide) : ReadInputStream(stream, filename) { //|------------------------------------------------------------------------------------- //| ensure 32/64 bit portability: used fixed size data types only e.g. boost::uint32_t | //|------------------------------------------------------------------------------------- boost::int32_t version = readNumberC(); - if (version != FILE_FORMAT_VER) //read file format version - throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + filename + wxT("\"")); - versionId = version; - //read DB id - const CharArray tmp = readArrayC(); - output.first.assign(tmp->begin(), tmp->end()); +#ifndef _MSC_VER +#warning remove this check after migration! +#endif + if (version != 6) //migrate! + + if (version != FILE_FORMAT_VER) //read file format version + throw FileError(_("Incompatible synchronization database format:") + " \n" + "\"" + filename.c_str() + "\""); + - DirectoryTOC& dbList = output.second; - dbList.clear(); +#ifndef _MSC_VER +#warning remove this case after migration! +#endif - boost::uint32_t dbCount = readNumberC(); //number of databases: one for each sync-pair - while (dbCount-- != 0) + if (version == 6) { - //DB id of partner databases - const CharArray tmp2 = readArrayC(); - const std::string partnerID(tmp2->begin(), tmp2->end()); + streamList.clear(); + + //read DB id + const CharArray tmp = readArrayC(); + std::string mainId(tmp->begin(), tmp->end()); + + boost::uint32_t dbCount = readNumberC(); //number of databases: one for each sync-pair + while (dbCount-- != 0) + { + //DB id of partner databases + const CharArray tmp2 = readArrayC(); + const std::string partnerID(tmp2->begin(), tmp2->end()); - CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation) + CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation) - dbList.insert(std::make_pair(partnerID, buffer)); + if (leftSide) + streamList.insert(std::make_pair(partnerID + mainId, buffer)); + else + streamList.insert(std::make_pair(mainId + partnerID, buffer)); + } + } + else + { + streamList.clear(); + + boost::uint32_t dbCount = readNumberC(); //number of databases: one for each sync-pair + while (dbCount-- != 0) + { + //DB id of partner databases + const CharArray tmp2 = readArrayC(); + const std::string sessionID(tmp2->begin(), tmp2->end()); + + CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation) + + streamList.insert(std::make_pair(sessionID, buffer)); + } } } }; +namespace +{ +StreamMapping loadStreams(const Zstring& filename, -DbStreamData loadFile(const Zstring& filename) //throw (FileError) +#ifndef _MSC_VER +#warning remove this parameter after migration! +#endif + bool leftSide) //throw (FileError) { if (!zen::fileExists(filename)) - throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + - _("One of the FreeFileSync database files is not yet existing:") + wxT(" \n") + - wxT("\"") + zToWx(filename) + wxT("\"")); + throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + " \n\n" + + _("One of the FreeFileSync database files is not yet existing:") + " \n" + + "\"" + filename + "\""); - //read format description (uncompressed) - FileInputStreamDB uncompressed(filename); //throw (FileError) + try + { + //read format description (uncompressed) + FileInputStreamDB uncompressed(filename); //throw (FileError) - wxZlibInputStream input(uncompressed, wxZLIB_ZLIB); + wxZlibInputStream input(uncompressed, wxZLIB_ZLIB); - DbStreamData output; - int versionId = 0; - ReadFileStream (input, zToWx(filename), output, versionId); - return output; + StreamMapping streamList; + ReadFileStream(input, toWx(filename), streamList, leftSide); + return streamList; + } + catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file + { + throw FileError(_("Error reading from synchronization database:") + " (bad_alloc)"); + } } -std::pair zen::loadFromDisk(const BaseDirMapping& baseMapping) //throw (FileError) +DirInfoPtr parseStream(const std::vector& stream, const Zstring& fileName) //throw FileError -> return value always bound! { - const Zstring fileNameLeft = baseMapping.getDBFilename(); - const Zstring fileNameRight = baseMapping.getDBFilename(); - try { - //read file data: db ID + mapping of partner-ID/DirInfo-stream - const DbStreamData dbEntriesLeft = ::loadFile(fileNameLeft); - const DbStreamData dbEntriesRight = ::loadFile(fileNameRight); - - //find associated DirInfo-streams - DirectoryTOC::const_iterator dbLeft = dbEntriesLeft.second.find(dbEntriesRight.first); //find left db-entry that corresponds to right database - DirectoryTOC::const_iterator dbRight = dbEntriesRight.second.find(dbEntriesLeft.first); //find left db-entry that corresponds to right database - - if (dbLeft == dbEntriesLeft.second.end()) - throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + - _("One of the FreeFileSync database entries within the following file is not yet existing:") + wxT(" \n") + - wxT("\"") + zToWx(fileNameLeft) + wxT("\"")); + //read streams into DirInfo + auto dirInfo = std::make_shared(); + wxMemoryInputStream buffer(&stream[0], stream.size()); //convert char-array to inputstream: no copying, ownership not transferred + ReadDirInfo(buffer, toWx(fileName), *dirInfo); //throw FileError + return dirInfo; + } + catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file + { + throw FileError(_("Error reading from synchronization database:") + " (bad_alloc)"); + } +} +} - if (dbRight == dbEntriesRight.second.end()) - throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + - _("One of the FreeFileSync database entries within the following file is not yet existing:") + wxT(" \n") + - wxT("\"") + zToWx(fileNameRight) + wxT("\"")); - //read streams into DirInfo - std::shared_ptr dirInfoLeft(new DirInformation); - wxMemoryInputStream buffer(&(*dbLeft->second)[0], dbLeft->second->size()); //convert char-array to inputstream: no copying, ownership not transferred - ReadDirInfo(buffer, zToWx(fileNameLeft), *dirInfoLeft); //read file/dir information +std::pair zen::loadFromDisk(const BaseDirMapping& baseMapping) //throw (FileError) +{ + const Zstring fileNameLeft = getDBFilename(baseMapping); + const Zstring fileNameRight = getDBFilename(baseMapping); - std::shared_ptr dirInfoRight(new DirInformation); - wxMemoryInputStream buffer2(&(*dbRight->second)[0], dbRight->second->size()); //convert char-array to inputstream: no copying, ownership not transferred - ReadDirInfo(buffer2, zToWx(fileNameRight), *dirInfoRight); //read file/dir information + //read file data: list of session ID + DirInfo-stream + const StreamMapping streamListLeft = ::loadStreams(fileNameLeft, true); //throw (FileError) + const StreamMapping streamListRight = ::loadStreams(fileNameRight, false); //throw (FileError) - return std::make_pair(dirInfoLeft, dirInfoRight); - } - catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file + //find associated session: there can be at most one session within intersection of left and right ids + StreamMapping::const_iterator streamLeft = streamListLeft .end(); + StreamMapping::const_iterator streamRight = streamListRight.end(); + for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft) { - throw FileError(wxString(_("Error reading from synchronization database:")) + wxT(" (bad_alloc)")); + auto iterRight = streamListRight.find(iterLeft->first); + if (iterRight != streamListRight.end()) + { + streamLeft = iterLeft; + streamRight = iterRight; + break; + } } + + if (streamLeft == streamListLeft .end() || + streamRight == streamListRight.end() || + !streamLeft ->second.get() || + !streamRight->second.get()) + throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + " \n\n" + + _("No matching synchronization session found in database files:") + " \n" + + "\"" + fileNameLeft + "\"\n" + + "\"" + fileNameRight + "\""); + //read streams into DirInfo + DirInfoPtr dirInfoLeft = parseStream(*streamLeft ->second, fileNameLeft); //throw FileError + DirInfoPtr dirInfoRight = parseStream(*streamRight->second, fileNameRight); //throw FileError + + return std::make_pair(dirInfoLeft, dirInfoRight); } @@ -375,21 +441,16 @@ private: class WriteFileStream : public WriteOutputStream { public: - WriteFileStream(const DbStreamData& input, const wxString& filename, wxOutputStream& stream) : WriteOutputStream(filename, stream) + WriteFileStream(const StreamMapping& streamList, const wxString& filename, wxOutputStream& stream) : WriteOutputStream(filename, stream) { //save file format version writeNumberC(FILE_FORMAT_VER); - //write DB id - writeArrayC(std::vector(input.first.begin(), input.first.end())); - - const DirectoryTOC& dbList = input.second; + writeNumberC(static_cast(streamList.size())); //number of database records: one for each sync-pair - writeNumberC(static_cast(dbList.size())); //number of database records: one for each sync-pair - - for (DirectoryTOC::const_iterator i = dbList.begin(); i != dbList.end(); ++i) + for (StreamMapping::const_iterator i = streamList.begin(); i != streamList.end(); ++i) { - //DB id of partner database + //sync session id writeArrayC(std::vector(i->first.begin(), i->first.end())); //write DirInformation stream @@ -400,7 +461,7 @@ public: //save/load DirContainer -void saveFile(const DbStreamData& dbStream, const Zstring& filename) //throw (FileError) +void saveFile(const StreamMapping& streamList, const Zstring& filename) //throw (FileError) { { //write format description (uncompressed) @@ -414,7 +475,7 @@ void saveFile(const DbStreamData& dbStream, const Zstring& filename) //throw (Fi 6 1,77 MB - 613 ms 9 (maximal compression) 1,74 MB - 3330 ms */ - WriteFileStream(dbStream, zToWx(filename), output); + WriteFileStream(streamList, toWx(filename), output); } //(try to) hide database file #ifdef FFS_WIN @@ -423,133 +484,132 @@ void saveFile(const DbStreamData& dbStream, const Zstring& filename) //throw (Fi } -bool entryExisting(const DirectoryTOC& table, const UniqueId& newKey, const MemoryStreamPtr& newValue) +bool equalEntry(const MemoryStreamPtr& lhs, const MemoryStreamPtr& rhs) { - DirectoryTOC::const_iterator iter = table.find(newKey); - if (iter == table.end()) - return false; - - if (!newValue.get() || !iter->second.get()) - return newValue.get() == iter->second.get(); + if (!lhs.get() || !rhs.get()) + return lhs.get() == rhs.get(); - return *newValue == *iter->second; + return *lhs == *rhs; } void zen::saveToDisk(const BaseDirMapping& baseMapping) //throw (FileError) { //transactional behaviour! write to tmp files first - const Zstring fileNameLeftTmp = baseMapping.getDBFilename() + Zstr(".tmp"); - const Zstring fileNameRightTmp = baseMapping.getDBFilename() + Zstr(".tmp");; + const Zstring dbNameLeftTmp = getDBFilename(baseMapping, true); + const Zstring dbNameRightTmp = getDBFilename(baseMapping, true); + + const Zstring dbNameLeft = getDBFilename(baseMapping); + const Zstring dbNameRight = getDBFilename(baseMapping); //delete old tmp file, if necessary -> throws if deletion fails! - removeFile(fileNameLeftTmp); // - removeFile(fileNameRightTmp); //throw (FileError) + removeFile(dbNameLeftTmp); // + removeFile(dbNameRightTmp); //throw (FileError) - //load old database files... + //(try to) load old database files... + StreamMapping streamListLeft; + StreamMapping streamListRight; - //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw! - DbStreamData dbEntriesLeft; - try - { - dbEntriesLeft = ::loadFile(baseMapping.getDBFilename()); - } - catch(FileError&) + try //read file data: list of session ID + DirInfo-stream { - //if error occurs: just overwrite old file! User is already informed about issues right after comparing! - //dbEntriesLeft has empty mapping, but already a DB-ID! - dbEntriesLeft.first = util::generateGUID(); + streamListLeft = ::loadStreams(dbNameLeft, true); } - - //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw! - DbStreamData dbEntriesRight; + catch(FileError&) {} //if error occurs: just overwrite old file! User is already informed about issues right after comparing! try { - dbEntriesRight = ::loadFile(baseMapping.getDBFilename()); + streamListRight = ::loadStreams(dbNameRight, false); } - catch(FileError&) + catch(FileError&) {} + + //find associated session: there can be at most one session within intersection of left and right ids + StreamMapping::iterator streamLeft = streamListLeft .end(); + StreamMapping::iterator streamRight = streamListRight.end(); + for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft) { - dbEntriesRight.first = util::generateGUID(); + auto iterRight = streamListRight.find(iterLeft->first); + if (iterRight != streamListRight.end()) + { + streamLeft = iterLeft; + streamRight = iterRight; + break; + } } - //(try to) read old DirInfo - std::shared_ptr oldDirInfoLeft; + DirInfoPtr oldDirInfoLeft; + DirInfoPtr oldDirInfoRight; try { - DirectoryTOC::const_iterator iter = dbEntriesLeft.second.find(dbEntriesRight.first); - if (iter != dbEntriesLeft.second.end()) - if (iter->second.get()) - { - const std::vector& memStream = *iter->second; - wxMemoryInputStream buffer(&memStream[0], memStream.size()); //convert char-array to inputstream: no copying, ownership not transferred - std::shared_ptr dirInfoTmp = std::make_shared(); - ReadDirInfo(buffer, zToWx(baseMapping.getDBFilename()), *dirInfoTmp); //read file/dir information - oldDirInfoLeft = dirInfoTmp; - } + if (streamLeft != streamListLeft .end() && + streamRight != streamListRight.end() && + streamLeft ->second.get() && + streamRight->second.get()) + { + oldDirInfoLeft = parseStream(*streamLeft ->second, dbNameLeft); //throw FileError + oldDirInfoRight = parseStream(*streamRight->second, dbNameRight); //throw FileError + } } - catch(FileError&) {} //if error occurs: just overwrite old file! User is already informed about issues right after comparing! - - std::shared_ptr oldDirInfoRight; - try + catch(FileError&) { - DirectoryTOC::const_iterator iter = dbEntriesRight.second.find(dbEntriesLeft.first); - if (iter != dbEntriesRight.second.end()) - if (iter->second.get()) - { - const std::vector& memStream = *iter->second; - wxMemoryInputStream buffer(&memStream[0], memStream.size()); //convert char-array to inputstream: no copying, ownership not transferred - std::shared_ptr dirInfoTmp = std::make_shared(); - ReadDirInfo(buffer, zToWx(baseMapping.getDBFilename()), *dirInfoTmp); //read file/dir information - oldDirInfoRight = dirInfoTmp; - } + //if error occurs: just overwrite old file! User is already informed about issues right after comparing! + oldDirInfoLeft .reset(); //read both or none! + oldDirInfoRight.reset(); // } - catch(FileError&) {} - //create new database entries - MemoryStreamPtr dbEntryLeft(new std::vector); + MemoryStreamPtr newStreamLeft = std::make_shared>(); { wxMemoryOutputStream buffer; - DirContainer* oldDir = oldDirInfoLeft.get() ? &oldDirInfoLeft->baseDirContainer : NULL; - SaveDirInfo(baseMapping, oldDir, zToWx(baseMapping.getDBFilename()), buffer); - dbEntryLeft->resize(buffer.GetSize()); //convert output stream to char-array - buffer.CopyTo(&(*dbEntryLeft)[0], buffer.GetSize()); // + const DirContainer* oldDir = oldDirInfoLeft.get() ? &oldDirInfoLeft->baseDirContainer : NULL; + SaveDirInfo(baseMapping, oldDir, toWx(dbNameLeft), buffer); + newStreamLeft->resize(buffer.GetSize()); //convert output stream to char-array + buffer.CopyTo(&(*newStreamLeft)[0], buffer.GetSize()); // } - MemoryStreamPtr dbEntryRight(new std::vector); + MemoryStreamPtr newStreamRight = std::make_shared>(); { wxMemoryOutputStream buffer; - DirContainer* oldDir = oldDirInfoRight.get() ? &oldDirInfoRight->baseDirContainer : NULL; - SaveDirInfo(baseMapping, oldDir, zToWx(baseMapping.getDBFilename()), buffer); - dbEntryRight->resize(buffer.GetSize()); //convert output stream to char-array - buffer.CopyTo(&(*dbEntryRight)[0], buffer.GetSize()); // + const DirContainer* oldDir = oldDirInfoRight.get() ? &oldDirInfoRight->baseDirContainer : NULL; + SaveDirInfo(baseMapping, oldDir, toWx(dbNameRight), buffer); + newStreamRight->resize(buffer.GetSize()); //convert output stream to char-array + buffer.CopyTo(&(*newStreamRight)[0], buffer.GetSize()); // } - //create/update DirInfo-streams + //check if there is some work to do at all { - const bool updateRequiredLeft = !entryExisting(dbEntriesLeft. second, dbEntriesRight.first, dbEntryLeft); - const bool updateRequiredRight = !entryExisting(dbEntriesRight.second, dbEntriesLeft. first, dbEntryRight); + const bool updateRequiredLeft = streamLeft == streamListLeft .end() || !equalEntry(newStreamLeft, streamLeft ->second); + const bool updateRequiredRight = streamRight == streamListRight.end() || !equalEntry(newStreamRight, streamRight->second); //some users monitor the *.ffs_db file with RTS => don't touch the file if it isnt't strictly needed if (!updateRequiredLeft && !updateRequiredRight) return; } - dbEntriesLeft .second[dbEntriesRight.first] = dbEntryLeft; - dbEntriesRight.second[dbEntriesLeft .first] = dbEntryRight; + + //create/update DirInfo-streams + std::string sessionID = util::generateGUID(); + + //erase old session data + if (streamLeft != streamListLeft.end()) + streamListLeft.erase(streamLeft); + if (streamRight != streamListRight.end()) + streamListRight.erase(streamRight); + + //fill in new + streamListLeft .insert(std::make_pair(sessionID, newStreamLeft)); + streamListRight.insert(std::make_pair(sessionID, newStreamRight)); //write (temp-) files... - Loki::ScopeGuard guardTempFileLeft = Loki::MakeGuard(&zen::removeFile, fileNameLeftTmp); - saveFile(dbEntriesLeft, fileNameLeftTmp); //throw (FileError) + Loki::ScopeGuard guardTempFileLeft = Loki::MakeGuard(&zen::removeFile, dbNameLeftTmp); + saveFile(streamListLeft, dbNameLeftTmp); //throw (FileError) - Loki::ScopeGuard guardTempFileRight = Loki::MakeGuard(&zen::removeFile, fileNameRightTmp); - saveFile(dbEntriesRight, fileNameRightTmp); //throw (FileError) + Loki::ScopeGuard guardTempFileRight = Loki::MakeGuard(&zen::removeFile, dbNameRightTmp); + saveFile(streamListRight, dbNameRightTmp); //throw (FileError) //operation finished: rename temp files -> this should work transactionally: //if there were no write access, creation of temp files would have failed - removeFile(baseMapping.getDBFilename()); - removeFile(baseMapping.getDBFilename()); - renameFile(fileNameLeftTmp, baseMapping.getDBFilename()); //throw (FileError); - renameFile(fileNameRightTmp, baseMapping.getDBFilename()); //throw (FileError); + removeFile(dbNameLeft); + removeFile(dbNameRight); + renameFile(dbNameLeftTmp, dbNameLeft); //throw (FileError); + renameFile(dbNameRightTmp, dbNameRight); //throw (FileError); guardTempFileLeft. Dismiss(); //no need to delete temp file anymore guardTempFileRight.Dismiss(); // -- cgit