// ************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** // #include "db_file.h" #include #include #include "../shared/global_func.h" #include "../shared/file_error.h" #include "../shared/string_conv.h" #include "../shared/file_handling.h" #include #include "../shared/serialize.h" #include "../shared/file_io.h" #include "../shared/loki/ScopeGuard.h" #include "../shared/i18n.h" #ifdef FFS_WIN #include //includes "windows.h" #include "../shared/long_path_prefix.h" #endif using namespace ffs3; namespace { //------------------------------------------------------------------------------------------------------------------------------- const char FILE_FORMAT_DESCR[] = "FreeFileSync"; const int FILE_FORMAT_VER = 5; //------------------------------------------------------------------------------------------------------------------------------- class FileInputStreamDB : public FileInputStream { public: FileInputStreamDB(const Zstring& filename) : //throw (FileError) FileInputStream(filename) { //read FreeFileSync file identifier char formatDescr[sizeof(FILE_FORMAT_DESCR)] = {}; Read(formatDescr, sizeof(formatDescr)); //throw (FileError) if (!std::equal(FILE_FORMAT_DESCR, FILE_FORMAT_DESCR + sizeof(FILE_FORMAT_DESCR), formatDescr)) throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + zToWx(filename) + wxT("\"")); } private: }; class FileOutputStreamDB : public FileOutputStream { public: FileOutputStreamDB(const Zstring& filename) : //throw (FileError) FileOutputStream(filename) { //write FreeFileSync file identifier Write(FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR)); //throw (FileError) } private: }; } //####################################################################################################################################### class ReadDirInfo : public util::ReadInputStream { public: ReadDirInfo(wxInputStream& stream, const wxString& errorObjName, DirInformation& dirInfo) : ReadInputStream(stream, errorObjName) { //|------------------------------------------------------------------------------------- //| ensure 32/64 bit portability: used fixed size data types only e.g. boost::uint32_t | //|------------------------------------------------------------------------------------- //read filter settings dirInfo.filter = BaseFilter::loadFilter(getStream()); check(); //start recursion execute(dirInfo.baseDirContainer); } private: void execute(DirContainer& dirCont) const { boost::uint32_t fileCount = readNumberC(); while (fileCount-- != 0) readSubFile(dirCont); boost::uint32_t symlinkCount = readNumberC(); while (symlinkCount-- != 0) readSubLink(dirCont); boost::uint32_t dirCount = readNumberC(); while (dirCount-- != 0) readSubDirectory(dirCont); } void readSubFile(DirContainer& dirCont) const { //attention: order of function argument evaluation is undefined! So do it one after the other... const Zstring shortName = readStringC(); //file name const boost::int32_t modHigh = readNumberC(); const boost::uint32_t modLow = readNumberC(); const boost::uint32_t sizeHigh = readNumberC(); const boost::uint32_t sizeLow = readNumberC(); //const util::FileID fileIdentifier(stream_); //check(); dirCont.addSubFile(shortName, FileDescriptor(wxLongLong(modHigh, modLow), wxULongLong(sizeHigh, sizeLow))); } void readSubLink(DirContainer& dirCont) const { //attention: order of function argument evaluation is undefined! So do it one after the other... const Zstring shortName = readStringC(); //file name const boost::int32_t modHigh = readNumberC(); const boost::uint32_t modLow = readNumberC(); const Zstring targetPath = readStringC(); //file name const LinkDescriptor::LinkType linkType = static_cast(readNumberC()); dirCont.addSubLink(shortName, LinkDescriptor(wxLongLong(modHigh, modLow), targetPath, linkType)); } void readSubDirectory(DirContainer& dirCont) const { const Zstring shortName = readStringC(); //directory name DirContainer& subDir = dirCont.addSubDir(shortName); execute(subDir); //recurse } }; namespace { typedef std::string UniqueId; typedef boost::shared_ptr > MemoryStreamPtr; //byte stream representing DirInformation typedef std::map DirectoryTOC; //list of streams ordered by a UUID pointing to their partner database typedef std::pair DbStreamData; //header data: UUID representing this database, item data: list of dir-streams } /* Example left side right side --------- ---------- DB-ID 123 <-\ /-> DB-ID 567 \/ Partner-ID 111 /\ Partner-ID 222 Partner-ID 567 _/ \_ Partner-ID 123 ... ... */ class ReadFileStream : public util::ReadInputStream { public: ReadFileStream(wxInputStream& stream, const wxString& filename, DbStreamData& output) : ReadInputStream(stream, filename) { //|------------------------------------------------------------------------------------- //| ensure 32/64 bit portability: used fixed size data types only e.g. boost::uint32_t | //|------------------------------------------------------------------------------------- if (readNumberC() != FILE_FORMAT_VER) //read file format version throw FileError(wxString(_("Incompatible synchronization database format:")) + wxT(" \n") + wxT("\"") + filename + wxT("\"")); //read DB id const CharArray tmp = readArrayC(); output.first.assign(tmp->begin(), tmp->end()); DirectoryTOC& dbList = output.second; dbList.clear(); boost::uint32_t dbCount = readNumberC(); //number of databases: one for each sync-pair while (dbCount-- != 0) { //DB id of partner databases const CharArray tmp2 = readArrayC(); const std::string partnerID(tmp2->begin(), tmp2->end()); CharArray buffer = readArrayC(); //read db-entry stream (containing DirInformation) dbList.insert(std::make_pair(partnerID, buffer)); } } }; DbStreamData loadFile(const Zstring& filename) //throw (FileError) { if (!ffs3::fileExists(filename)) throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + _("One of the FreeFileSync database files is not yet existing:") + wxT(" \n") + wxT("\"") + zToWx(filename) + wxT("\"")); //read format description (uncompressed) FileInputStreamDB uncompressed(filename); //throw (FileError) wxZlibInputStream input(uncompressed, wxZLIB_ZLIB); DbStreamData output; ReadFileStream(input, zToWx(filename), output); return output; } std::pair ffs3::loadFromDisk(const BaseDirMapping& baseMapping) //throw (FileError) { const Zstring fileNameLeft = baseMapping.getDBFilename(); const Zstring fileNameRight = baseMapping.getDBFilename(); try { //read file data: db ID + mapping of partner-ID/DirInfo-stream const DbStreamData dbEntriesLeft = ::loadFile(fileNameLeft); const DbStreamData dbEntriesRight = ::loadFile(fileNameRight); //find associated DirInfo-streams DirectoryTOC::const_iterator dbLeft = dbEntriesLeft.second.find(dbEntriesRight.first); //find left db-entry that corresponds to right database DirectoryTOC::const_iterator dbRight = dbEntriesRight.second.find(dbEntriesLeft.first); //find left db-entry that corresponds to right database if (dbLeft == dbEntriesLeft.second.end()) throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + _("One of the FreeFileSync database entries within the following file is not yet existing:") + wxT(" \n") + wxT("\"") + zToWx(fileNameLeft) + wxT("\"")); if (dbRight == dbEntriesRight.second.end()) throw FileErrorDatabaseNotExisting(wxString(_("Initial synchronization:")) + wxT(" \n\n") + _("One of the FreeFileSync database entries within the following file is not yet existing:") + wxT(" \n") + wxT("\"") + zToWx(fileNameRight) + wxT("\"")); //read streams into DirInfo boost::shared_ptr dirInfoLeft(new DirInformation); wxMemoryInputStream buffer(&(*dbLeft->second)[0], dbLeft->second->size()); //convert char-array to inputstream: no copying, ownership not transferred ReadDirInfo(buffer, zToWx(fileNameLeft), *dirInfoLeft); //read file/dir information boost::shared_ptr dirInfoRight(new DirInformation); wxMemoryInputStream buffer2(&(*dbRight->second)[0], dbRight->second->size()); //convert char-array to inputstream: no copying, ownership not transferred ReadDirInfo(buffer2, zToWx(fileNameRight), *dirInfoRight); //read file/dir information return std::make_pair(dirInfoLeft, dirInfoRight); } catch (const std::bad_alloc&) //this is most likely caused by a corrupted database file { throw FileError(wxString(_("Error reading from synchronization database:")) + wxT(" (bad_alloc)")); } } //------------------------------------------------------------------------------------------------------------------------- template struct IsNonEmpty { bool operator()(const FileSystemObject& fsObj) const { return !fsObj.isEmpty(); } }; template class SaveDirInfo : public util::WriteOutputStream { public: SaveDirInfo(const BaseDirMapping& baseMapping, const wxString& errorObjName, wxOutputStream& stream) : WriteOutputStream(errorObjName, stream) { //save filter settings baseMapping.getFilter()->saveFilter(getStream()); check(); //start recursion execute(baseMapping); } private: friend class util::ProxyForEach >; //friend declaration of std::for_each is NOT sufficient as implementation is compiler dependent! void execute(const HierarchyObject& hierObj) { util::ProxyForEach > prx(*this); //grant std::for_each access to private parts of this class writeNumberC(std::count_if(hierObj.useSubFiles().begin(), hierObj.useSubFiles().end(), IsNonEmpty())); //number of (existing) files std::for_each(hierObj.useSubFiles().begin(), hierObj.useSubFiles().end(), prx); writeNumberC(std::count_if(hierObj.useSubLinks().begin(), hierObj.useSubLinks().end(), IsNonEmpty())); //number of (existing) files std::for_each(hierObj.useSubLinks().begin(), hierObj.useSubLinks().end(), prx); writeNumberC(std::count_if(hierObj.useSubDirs().begin(), hierObj.useSubDirs().end(), IsNonEmpty())); //number of (existing) directories std::for_each(hierObj.useSubDirs().begin(), hierObj.useSubDirs().end(), prx); } void operator()(const FileMapping& fileMap) { if (!fileMap.isEmpty()) { writeStringC(fileMap.getShortName()); //file name writeNumberC (fileMap.getLastWriteTime().GetHi()); //last modification time writeNumberC(fileMap.getLastWriteTime().GetLo()); // writeNumberC(fileMap.getFileSize().GetHi()); //filesize writeNumberC(fileMap.getFileSize().GetLo()); // //fileMap.getFileID().toStream(stream_); //unique file identifier //check(); } } void operator()(const SymLinkMapping& linkObj) { if (!linkObj.isEmpty()) { writeStringC(linkObj.getShortName()); writeNumberC (linkObj.getLastWriteTime().GetHi()); //last modification time writeNumberC(linkObj.getLastWriteTime().GetLo()); // writeStringC(linkObj.getTargetPath()); writeNumberC(linkObj.getLinkType()); } } void operator()(const DirMapping& dirMap) { if (!dirMap.isEmpty()) { writeStringC(dirMap.getShortName()); //directory name execute(dirMap); //recurse } } }; class WriteFileStream : public util::WriteOutputStream { public: WriteFileStream(const DbStreamData& input, const wxString& filename, wxOutputStream& stream) : WriteOutputStream(filename, stream) { //save file format version writeNumberC(FILE_FORMAT_VER); //write DB id writeArrayC(std::vector(input.first.begin(), input.first.end())); const DirectoryTOC& dbList = input.second; writeNumberC(static_cast(dbList.size())); //number of database records: one for each sync-pair for (DirectoryTOC::const_iterator i = dbList.begin(); i != dbList.end(); ++i) { //DB id of partner database writeArrayC(std::vector(i->first.begin(), i->first.end())); //write DirInformation stream writeArrayC(*(i->second)); } } }; //save/load DirContainer void saveFile(const DbStreamData& dbStream, const Zstring& filename) //throw (FileError) { { //write format description (uncompressed) FileOutputStreamDB uncompressed(filename); //throw (FileError) wxZlibOutputStream output(uncompressed, 4, wxZLIB_ZLIB); /* 4 - best compromise between speed and compression: (scanning 200.000 objects) 0 (uncompressed) 8,95 MB - 422 ms 2 2,07 MB - 470 ms 4 1,87 MB - 500 ms 6 1,77 MB - 613 ms 9 (maximal compression) 1,74 MB - 3330 ms */ WriteFileStream(dbStream, zToWx(filename), output); } //(try to) hide database file #ifdef FFS_WIN ::SetFileAttributes(ffs3::applyLongPathPrefix(filename).c_str(), FILE_ATTRIBUTE_HIDDEN); #endif } bool entryExisting(const DirectoryTOC& table, const UniqueId& newKey, const MemoryStreamPtr& newValue) { DirectoryTOC::const_iterator iter = table.find(newKey); if (iter == table.end()) return false; if (!newValue.get() || !iter->second.get()) return !newValue.get() && !iter->second.get(); return *newValue == *iter->second; } void ffs3::saveToDisk(const BaseDirMapping& baseMapping) //throw (FileError) { //transactional behaviour! write to tmp files first const Zstring fileNameLeftTmp = baseMapping.getDBFilename() + Zstr(".tmp"); const Zstring fileNameRightTmp = baseMapping.getDBFilename() + Zstr(".tmp");; //delete old tmp file, if necessary -> throws if deletion fails! removeFile(fileNameLeftTmp); // removeFile(fileNameRightTmp); //throw (FileError) //load old database files... //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw! DbStreamData dbEntriesLeft; try { dbEntriesLeft = ::loadFile(baseMapping.getDBFilename()); } catch(FileError&) { //if error occurs: just overwrite old file! User is already informed about issues right after comparing! //dbEntriesLeft has empty mapping, but already a DB-ID! dbEntriesLeft.first = util::generateGUID(); } //read file data: db ID + mapping of partner-ID/DirInfo-stream: may throw! DbStreamData dbEntriesRight; try { dbEntriesRight = ::loadFile(baseMapping.getDBFilename()); } catch(FileError&) { dbEntriesRight.first = util::generateGUID(); } //create new database entries MemoryStreamPtr dbEntryLeft(new std::vector); { wxMemoryOutputStream buffer; SaveDirInfo(baseMapping, zToWx(baseMapping.getDBFilename()), buffer); dbEntryLeft->resize(buffer.GetSize()); //convert output stream to char-array buffer.CopyTo(&(*dbEntryLeft)[0], buffer.GetSize()); // } MemoryStreamPtr dbEntryRight(new std::vector); { wxMemoryOutputStream buffer; SaveDirInfo(baseMapping, zToWx(baseMapping.getDBFilename()), buffer); dbEntryRight->resize(buffer.GetSize()); //convert output stream to char-array buffer.CopyTo(&(*dbEntryRight)[0], buffer.GetSize()); // } //create/update DirInfo-streams { const bool updateRequiredLeft = !entryExisting(dbEntriesLeft. second, dbEntriesRight.first, dbEntryLeft); const bool updateRequiredRight = !entryExisting(dbEntriesRight.second, dbEntriesLeft. first, dbEntryRight); //some users monitor the *.ffs_db file with RTS => don't touch the file if it isnt't strictly needed if (!updateRequiredLeft && !updateRequiredRight) return; } dbEntriesLeft.second[dbEntriesRight.first] = dbEntryLeft; dbEntriesRight.second[dbEntriesLeft.first] = dbEntryRight; //write (temp-) files... Loki::ScopeGuard guardTempFileLeft = Loki::MakeGuard(&ffs3::removeFile, fileNameLeftTmp); saveFile(dbEntriesLeft, fileNameLeftTmp); //throw (FileError) Loki::ScopeGuard guardTempFileRight = Loki::MakeGuard(&ffs3::removeFile, fileNameRightTmp); saveFile(dbEntriesRight, fileNameRightTmp); //throw (FileError) //operation finished: rename temp files -> this should work transactionally: //if there were no write access, creation of temp files would have failed removeFile(baseMapping.getDBFilename()); removeFile(baseMapping.getDBFilename()); renameFile(fileNameLeftTmp, baseMapping.getDBFilename()); //throw (FileError); renameFile(fileNameRightTmp, baseMapping.getDBFilename()); //throw (FileError); guardTempFileLeft. Dismiss(); //no need to delete temp file anymore guardTempFileRight.Dismiss(); // }