diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/cmp_filetime.h | 9 | ||||
| -rw-r--r-- | lib/db_file.cpp | 994 | ||||
| -rw-r--r-- | lib/db_file.h | 68 | ||||
| -rw-r--r-- | lib/dir_lock.cpp | 77 | ||||
| -rw-r--r-- | lib/ffs_paths.h | 4 | ||||
| -rw-r--r-- | lib/generate_logfile.h | 138 | ||||
| -rw-r--r-- | lib/hard_filter.cpp | 92 | ||||
| -rw-r--r-- | lib/hard_filter.h | 98 | ||||
| -rw-r--r-- | lib/icon_buffer.cpp | 38 | ||||
| -rw-r--r-- | lib/localization.cpp | 3 | ||||
| -rw-r--r-- | lib/parallel_scan.cpp | 27 | ||||
| -rw-r--r-- | lib/process_xml.cpp | 40 | ||||
| -rw-r--r-- | lib/resolve_path.cpp | 7 | ||||
| -rw-r--r-- | lib/resources.cpp | 2 | ||||
| -rw-r--r-- | lib/xml_base.cpp | 2 |
15 files changed, 1100 insertions, 499 deletions
diff --git a/lib/cmp_filetime.h b/lib/cmp_filetime.h index afc97b9d..eb595ace 100644 --- a/lib/cmp_filetime.h +++ b/lib/cmp_filetime.h @@ -24,8 +24,6 @@ static const long oneYearFromNow = wxGetUTCTime() + 365 * 24 * 3600; //init at p class CmpFileTime { public: - CmpFileTime(size_t tolerance) : tolerance_(tolerance) {} - enum Result { TIME_EQUAL, @@ -35,9 +33,9 @@ public: TIME_RIGHT_INVALID }; - Result getResult(const Int64& lhs, const Int64& rhs) const + static Result getResult(const Int64& lhs, const Int64& rhs, size_t tolerance) { - if (sameFileTime(lhs, rhs, tolerance_)) //last write time may differ by up to 2 seconds (NTFS vs FAT32) + if (sameFileTime(lhs, rhs, tolerance)) //last write time may differ by up to 2 seconds (NTFS vs FAT32) return TIME_EQUAL; //check for erroneous dates @@ -53,9 +51,6 @@ public: else return TIME_LEFT_NEWER; } - -private: - const size_t tolerance_; }; } diff --git a/lib/db_file.cpp b/lib/db_file.cpp index 33c83a0e..787325e2 100644 --- a/lib/db_file.cpp +++ b/lib/db_file.cpp @@ -5,17 +5,21 @@ // ************************************************************************** #include "db_file.h" -#include <wx/wfstream.h> -#include <wx/zstream.h> -#include <wx/mstream.h> #include <zen/file_error.h> -#include <wx+/string_conv.h> #include <zen/file_handling.h> -#include <wx+/serialize.h> -#include <zen/file_io.h> #include <zen/scope_guard.h> #include <zen/guid.h> #include <zen/utf.h> +#include <wx+/zlib_wrap.h> +#include <wx+/serialize.h> + +#ifdef FFS_WIN +warn_static("get rid of wx headers") +#endif +#include <wx/wfstream.h> +#include <wx/zstream.h> +#include <wx/mstream.h> +#include <zen/perf.h> #ifdef FFS_WIN #include <zen/win.h> //includes "windows.h" @@ -29,13 +33,11 @@ namespace { //------------------------------------------------------------------------------------------------------------------------------- const char FILE_FORMAT_DESCR[] = "FreeFileSync"; -const int FILE_FORMAT_VER = 8; +const int FILE_FORMAT_VER = 9; //------------------------------------------------------------------------------------------------------------------------------- typedef std::string UniqueId; -typedef Zbase<char> MemoryStream; //ref-counted byte stream representing DirInformation -typedef std::map<UniqueId, MemoryStream> StreamMapping; //list of streams ordered by session UUID - +typedef std::map<UniqueId, BinaryStream> StreamMapping; //list of streams ordered by session UUID //----------------------------------------------------------------------------------- //| ensure 32/64 bit portability: use fixed size data types only e.g. std::uint32_t | @@ -48,7 +50,7 @@ Zstring getDBFilename(const BaseDirMapping& baseMap, bool tempfile = false) //Linux and Windows builds are binary incompatible: different file id?, problem with case sensitivity? //however 32 and 64 bit db files *are* designed to be binary compatible! //Give db files different names. - //make sure they end with ".ffs_db". These files will not be included into comparison + //make sure they end with ".ffs_db". These files will be excluded from comparison #ifdef FFS_WIN Zstring dbname = Zstring(Zstr("sync")) + (tempfile ? Zstr(".tmp") : Zstr("")) + SYNC_DB_FILE_ENDING; #elif defined FFS_LINUX @@ -59,65 +61,77 @@ Zstring getDBFilename(const BaseDirMapping& baseMap, bool tempfile = false) return baseMap.getBaseDirPf<side>() + dbname; } +//####################################################################################################################################### -class CheckedDbReader : public CheckedReader +//save/load streams +void saveStreams(const StreamMapping& streamList, const Zstring& filename) //throw FileError { -public: - CheckedDbReader(wxInputStream& stream, const Zstring& errorObjName) : CheckedReader(stream), errorObjName_(errorObjName) {} + BinStreamOut streamOut; -private: - virtual void throwException() const { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(errorObjName_))); } + //write FreeFileSync file identifier + writeArray(streamOut, FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR)); - const Zstring errorObjName_; -}; + //save file format version + writeNumber<std::int32_t>(streamOut, FILE_FORMAT_VER); + //save stream list + writeNumber<std::uint32_t>(streamOut, static_cast<std::uint32_t>(streamList.size())); //number of streams, one for each sync-pair -class CheckedDbWriter : public CheckedWriter -{ -public: - CheckedDbWriter(wxOutputStream& stream, const Zstring& errorObjName) : CheckedWriter(stream), errorObjName_(errorObjName) {} + for (auto iter = streamList.begin(); iter != streamList.end(); ++iter) + { + writeContainer<std::string >(streamOut, iter->first ); + writeContainer<BinaryStream>(streamOut, iter->second); + } -private: - virtual void throwException() const { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(errorObjName_))); } + saveBinStream(filename, streamOut.get()); //throw FileError - const Zstring errorObjName_; -}; +#ifdef FFS_WIN + ::SetFileAttributes(applyLongPathPrefix(filename).c_str(), FILE_ATTRIBUTE_HIDDEN); //(try to) hide database file +#endif +} +#ifdef FFS_WIN +warn_static("remove after migration") +#endif -StreamMapping loadStreams(const Zstring& filename) //throw FileError +StreamMapping loadStreams_v8(const Zstring& filename); //throw FileError + + +StreamMapping loadStreams(const Zstring& filename) //throw FileError, FileErrorDatabaseNotExisting { try { - //read format description (uncompressed) - FileInputStream rawStream(filename); //throw FileError, ErrorNotExisting + BinStreamIn streamIn = loadBinStream<BinaryStream>(filename); //throw FileError, ErrorNotExisting //read FreeFileSync file identifier char formatDescr[sizeof(FILE_FORMAT_DESCR)] = {}; - rawStream.Read(formatDescr, sizeof(formatDescr)); //throw FileError + readArray(streamIn, formatDescr, sizeof(formatDescr)); //throw UnexpectedEndOfStreamError if (!std::equal(FILE_FORMAT_DESCR, FILE_FORMAT_DESCR + sizeof(FILE_FORMAT_DESCR), formatDescr)) throw FileError(replaceCpy(_("Database file %x is incompatible."), L"%x", fmtFileName(filename))); - wxZlibInputStream decompressed(rawStream, wxZLIB_ZLIB); + const int version = readNumber<std::int32_t>(streamIn); //throw UnexpectedEndOfStreamError + if (version != FILE_FORMAT_VER) //read file format version# + //throw FileError(replaceCpy(_("Database file %x is incompatible."), L"%x", fmtFileName(filename))); + return loadStreams_v8(filename); - CheckedDbReader cr(decompressed, filename); + #ifdef FFS_WIN +warn_static("fix after migration") +#endif - std::int32_t version = cr.readPOD<std::int32_t>(); - if (version != FILE_FORMAT_VER) //read file format version# - throw FileError(replaceCpy(_("Database file %x is incompatible."), L"%x", fmtFileName(filename))); //read stream lists StreamMapping output; - std::uint32_t dbCount = cr.readPOD<std::uint32_t>(); //number of databases: one for each sync-pair + size_t dbCount = readNumber<std::uint32_t>(streamIn); //number of streams, one for each sync-pair while (dbCount-- != 0) { //DB id of partner databases - const std::string sessionID = cr.readString<std::string>(); - const MemoryStream stream = cr.readString<MemoryStream>(); //read db-entry stream (containing DirInformation) + std::string sessionID = readContainer<std::string >(streamIn); //throw UnexpectedEndOfStreamError + BinaryStream stream = readContainer<BinaryStream>(streamIn); // - output.insert(std::make_pair(sessionID, stream)); + output[sessionID] = std::move(stream); } return output; } @@ -126,303 +140,553 @@ StreamMapping loadStreams(const Zstring& filename) //throw FileError throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + L" \n" + replaceCpy(_("Database file %x does not yet exist."), L"%x", fmtFileName(filename))); } - catch (const std::bad_alloc& e) + catch (UnexpectedEndOfStreamError&) { - throw FileError(_("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); + throw FileError(_("Database file is corrupt:") + L"\n" + fmtFileName(filename)); + } + catch (const std::bad_alloc& e) //still required? + { + throw FileError(_("Database file is corrupt:") + L"\n" + fmtFileName(filename) + L"\n\n" + + _("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); } } +//####################################################################################################################################### -class StreamParser : private CheckedDbReader +#ifdef FFS_WIN +warn_static("remove v8Compatibilty after migration") +#endif + +class StreamGenerator //for db-file back-wards compatibility we stick with two output streams until further { public: - static DirInfoPtr execute(const MemoryStream& stream, const Zstring& fileName) //throw FileError -> return value always bound! + static void execute(const InSyncDir& dir, //throw FileError + const Zstring& filenameL, //used for diagnostics only + const Zstring& filenameR, + BinaryStream& streamL, + BinaryStream& streamR, + bool v8Compatibilty) { - try + StreamGenerator generator; + + //PERF_START + generator.recurse(dir); + //PERF_STOP + + auto compStream = [](const BinaryStream& stream, const Zstring& filename) -> BinaryStream //throw FileError { - //read streams into DirInfo - auto dirInfo = std::make_shared<DirInformation>(); - wxMemoryInputStream buffer(&*stream.begin(), stream.size()); //convert char-array to inputstream: no copying, ownership not transferred - StreamParser(buffer, fileName, *dirInfo); //throw FileError - return dirInfo; - } - catch (const std::bad_alloc& e) + try + { + /* Zlib: optimal level - testcase 1 million files + level/size [MB]/time [ms] + 0 49.54 272 (uncompressed) + 1 14.53 1013 + 2 14.13 1106 + 3 13.76 1288 - best compromise between speed and compression + 4 13.20 1526 + 5 12.73 1916 + 6 12.58 2765 + 7 12.54 3633 + 8 12.51 9032 + 9 12.50 19698 (maximal compression) */ + return compress(stream, 3); //throw ZlibInternalError + } + catch (ZlibInternalError&) + { + throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(filename)) + L" (zlib error)"); + } + }; + + const BinaryStream tmpL = compStream(generator.outputLeft .get(), filenameL); + const BinaryStream tmpR = compStream(generator.outputRight.get(), filenameR); + const BinaryStream tmpB = compStream(generator.outputBoth .get(), filenameL + Zstr("/") + filenameR); + + //distribute "outputBoth" over left and right streams: + BinStreamOut outL; + BinStreamOut outR; + writeNumber<bool>(outL, true); //this side contains first part of "outputBoth" + writeNumber<bool>(outR, false); + + size_t size1stPart = tmpB.size() / 2; + size_t size2ndPart = tmpB.size() - size1stPart; + + if (v8Compatibilty) { - throw FileError(_("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); + size1stPart = tmpB.size(); + size2ndPart = 0; } + + writeNumber<std::uint64_t>(outL, size1stPart); + writeNumber<std::uint64_t>(outR, size2ndPart); + + writeArray(outL, &*tmpB.begin(), size1stPart); + writeArray(outR, &*tmpB.begin() + size1stPart, size2ndPart); + + //write streams corresponding to one side only + writeContainer<BinaryStream>(outL, tmpL); + writeContainer<BinaryStream>(outR, tmpR); + + streamL = outL.get(); + streamR = outR.get(); } private: - StreamParser(wxInputStream& stream, const Zstring& errorObjName, DirInformation& dirInfo) : CheckedDbReader(stream, errorObjName) + void recurse(const InSyncDir& container) { - recurse(dirInfo.baseDirContainer); + // for (const auto& filePair : container.files) { processFile(filePair); }); ! + + writeNumber<std::uint32_t>(outputBoth, static_cast<std::uint32_t>(container.files.size())); + std::for_each(container.files.begin(), container.files.end(), [&](const std::pair<Zstring, InSyncFile>& filePair) { this->process(filePair); }); + + writeNumber<std::uint32_t>(outputBoth, static_cast<std::uint32_t>(container.symlinks.size())); + std::for_each(container.symlinks.begin(), container.symlinks.end(), [&](const std::pair<Zstring, InSyncSymlink>& symlinkPair) { this->process(symlinkPair); }); + + writeNumber<std::uint32_t>(outputBoth, static_cast<std::uint32_t>(container.dirs.size())); + std::for_each(container.dirs.begin(), container.dirs.end(), [&](const std::pair<Zstring, InSyncDir>& dirPair) { this->process(dirPair); }); } - Zstring readStringUtf8() const + static void writeUtf8(BinStreamOut& output, const Zstring& str) { writeContainer(output, utfCvrtTo<Zbase<char>>(str)); } + + static void write(BinStreamOut& output, const FileDescriptor& descr) { - return utfCvrtTo<Zstring>(readString<Zbase<char>>()); + writeNumber<std:: int64_t>(output, to<std:: int64_t>(descr.lastWriteTimeRaw)); + writeNumber<std::uint64_t>(output, to<std::uint64_t>(descr.fileSize)); + writeNumber<std::uint64_t>(output, descr.id.first ); //device id + writeNumber<std::uint64_t>(output, descr.id.second); //file id + assert_static(sizeof(descr.id.first ) <= sizeof(std::uint64_t)); + assert_static(sizeof(descr.id.second) <= sizeof(std::uint64_t)); } - FileId readFileId() const + static void write(BinStreamOut& output, const LinkDescriptor& descr) { - assert_static(sizeof(FileId().first ) <= sizeof(std::uint64_t)); - assert_static(sizeof(FileId().second) <= sizeof(std::uint64_t)); + writeNumber<std::int64_t>(output, to<std:: int64_t>(descr.lastWriteTimeRaw)); + writeUtf8(output, descr.targetPath); + writeNumber<std::int32_t>(output, descr.type); + } - const auto deviceId = static_cast<decltype(FileId().first )>(readPOD<std::uint64_t>()); // - const auto fileId = static_cast<decltype(FileId().second)>(readPOD<std::uint64_t>()); //silence "loss of precision" compiler warnings - return std::make_pair(deviceId, fileId); + static void write(BinStreamOut& output, const InSyncDir::InSyncStatus& status) + { + writeNumber<std::int32_t>(output, status); } - void recurse(DirContainer& dirCont) const + void process(const std::pair<Zstring, InSyncFile>& filePair) { - while (readPOD<bool>()) //files - { - //attention: order of function argument evaluation is undefined! So do it one after the other... - const Zstring shortName = readStringUtf8(); //file name + writeUtf8(outputBoth, filePair.first); + writeNumber<std::int32_t>(outputBoth, filePair.second.inSyncType); - const std::int64_t modTime = readPOD<std::int64_t>(); - const std::uint64_t fileSize = readPOD<std::uint64_t>(); - const FileId fileID = readFileId(); + write(outputLeft, filePair.second.left); + write(outputRight, filePair.second.right); + } - dirCont.addSubFile(shortName, - FileDescriptor(modTime, fileSize, fileID)); - } + void process(const std::pair<Zstring, InSyncSymlink>& symlinkPair) + { + writeUtf8(outputBoth, symlinkPair.first); + write(outputLeft, symlinkPair.second.left); + write(outputRight, symlinkPair.second.right); + } - while (readPOD<bool>()) //symlinks - { - //attention: order of function argument evaluation is undefined! So do it one after the other... - const Zstring shortName = readStringUtf8(); //file name - const std::int64_t modTime = readPOD<std::int64_t>(); - const Zstring targetPath = readStringUtf8(); //file name - const LinkDescriptor::LinkType linkType = static_cast<LinkDescriptor::LinkType>(readPOD<std::int32_t>()); - - dirCont.addSubLink(shortName, - LinkDescriptor(modTime, targetPath, linkType)); - } + void process(const std::pair<Zstring, InSyncDir>& dirPair) + { + writeUtf8(outputBoth, dirPair.first); + write(outputBoth, dirPair.second.status); - while (readPOD<bool>()) //directories - { - const Zstring shortName = readStringUtf8(); //directory name - DirContainer& subDir = dirCont.addSubDir(shortName); - recurse(subDir); - } + recurse(dirPair.second); } + + BinStreamOut outputLeft; //data related to one side only + BinStreamOut outputRight; // + BinStreamOut outputBoth; //data concerning both sides }; -//save/load DirContainer -void saveFile(const StreamMapping& streamList, const Zstring& filename) //throw FileError +class StreamParser //for db-file back-wards compatibility we stick with two output streams until further { +public: + static std::shared_ptr<InSyncDir> execute(const BinaryStream& streamL, //throw FileError + const BinaryStream& streamR, + const Zstring& filenameL, //used for diagnostics only + const Zstring& filenameR) { - FileOutputStream rawStream(filename); //throw FileError + auto decompStream = [](const BinaryStream& stream, const Zstring& filename) -> BinaryStream //throw FileError + { + try + { + return decompress(stream); //throw ZlibInternalError + } + catch (ZlibInternalError&) + { + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(filename)) + L" (zlib error)"); + } + }; + + try + { + BinStreamIn inL(streamL); + BinStreamIn inR(streamR); + + bool has1stPartL = readNumber<bool>(inL); //throw UnexpectedEndOfStreamError + bool has1stPartR = readNumber<bool>(inR); // + +#ifdef FFS_WIN +warn_static("restore check after migration!") +#endif - //write FreeFileSync file identifier - rawStream.Write(FILE_FORMAT_DESCR, sizeof(FILE_FORMAT_DESCR)); //throw FileError + //if (has1stPartL == has1stPartR) + // throw UnexpectedEndOfStreamError(); - wxZlibOutputStream compressed(rawStream, 4, wxZLIB_ZLIB); - /* 4 - best compromise between speed and compression: (scanning 200.000 objects) - 0 (uncompressed) 8,95 MB - 422 ms - 2 2,07 MB - 470 ms - 4 1,87 MB - 500 ms - 6 1,77 MB - 613 ms - 9 (maximal compression) 1,74 MB - 3330 ms */ + BinStreamIn& in1stPart = has1stPartL ? inL : inR; + BinStreamIn& in2ndPart = has1stPartL ? inR : inL; - CheckedDbWriter cw(compressed, filename); + const size_t size1stPart = readNumber<std::uint64_t>(in1stPart); + const size_t size2ndPart = readNumber<std::uint64_t>(in2ndPart); - //save file format version - cw.writePOD<std::int32_t>(FILE_FORMAT_VER); + BinaryStream tmpB; + tmpB.resize(size1stPart + size2ndPart); + readArray(in1stPart, &*tmpB.begin(), size1stPart); + readArray(in2ndPart, &*tmpB.begin() + size1stPart, size2ndPart); - //save stream list - cw.writePOD<std::uint32_t>(static_cast<std::uint32_t>(streamList.size())); //number of database records: one for each sync-pair + const BinaryStream tmpL = readContainer<BinaryStream>(inL); + const BinaryStream tmpR = readContainer<BinaryStream>(inR); - for (auto iter = streamList.begin(); iter != streamList.end(); ++iter) + auto output = std::make_shared<InSyncDir>(InSyncDir::STATUS_IN_SYNC); + StreamParser(decompStream(tmpL, filenameL), + decompStream(tmpR, filenameR), + decompStream(tmpB, filenameL + Zstr("/") + filenameR), + *output); //throw UnexpectedEndOfStreamError + return output; + } + catch (const UnexpectedEndOfStreamError&) { - cw.writeString<std::string >(iter->first ); //sync session id - cw.writeString<MemoryStream>(iter->second); //DirInformation stream + throw FileError(_("Database file is corrupt:") + L"\n" + fmtFileName(filenameL) + L"\n" + fmtFileName(filenameR)); + } + catch (const std::bad_alloc& e) //still required? + { + throw FileError(_("Database file is corrupt:") + L"\n" + fmtFileName(filenameL) + L"\n" + fmtFileName(filenameR) + L"\n\n" + + _("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); } } -#ifdef FFS_WIN - ::SetFileAttributes(applyLongPathPrefix(filename).c_str(), FILE_ATTRIBUTE_HIDDEN); //(try to) hide database file -#endif -} +private: + StreamParser(const BinaryStream& bufferL, + const BinaryStream& bufferR, + const BinaryStream& bufferB, + InSyncDir& container) : + inputLeft (bufferL), + inputRight(bufferR), + inputBoth (bufferB) { recurse(container); } -template <SelectedSide side> -class StreamGenerator : private CheckedDbWriter -{ -public: - static MemoryStream execute(const BaseDirMapping& baseMapping, const DirContainer* oldDirInfo, const Zstring& errorObjName) + static Zstring readUtf8(BinStreamIn& input) { return utfCvrtTo<Zstring>(readContainer<Zbase<char>>(input)); } //throw UnexpectedEndOfStreamError + + static void read(BinStreamIn& input, FileDescriptor& descr) { - wxMemoryOutputStream buffer; - StreamGenerator(baseMapping, oldDirInfo, errorObjName, buffer); + //attention: order of function argument evaluation is undefined! So do it one after the other... + descr.lastWriteTimeRaw = readNumber<std::int64_t>(input); //throw UnexpectedEndOfStreamError + descr.fileSize = readNumber<std::uint64_t>(input); + descr.id.first = static_cast<decltype(descr.id.first )>(readNumber<std::uint64_t>(input)); // + descr.id.second = static_cast<decltype(descr.id.second)>(readNumber<std::uint64_t>(input)); //silence "loss of precision" compiler warnings + } - MemoryStream output; - output.resize(buffer.GetSize()); - buffer.CopyTo(&*output.begin(), buffer.GetSize()); - return output; + static void read(BinStreamIn& input, LinkDescriptor& descr) + { + descr.lastWriteTimeRaw = readNumber<std::int64_t>(input); + descr.targetPath = readUtf8(input); //file name + descr.type = static_cast<LinkDescriptor::LinkType>(readNumber<std::int32_t>(input)); } -private: - StreamGenerator(const BaseDirMapping& baseMapping, const DirContainer* oldDirInfo, const Zstring& errorObjName, wxOutputStream& stream) : CheckedDbWriter(stream, errorObjName) + static void read(BinStreamIn& input, InSyncDir::InSyncStatus& status) { - recurse(baseMapping, oldDirInfo); + status = static_cast<InSyncDir::InSyncStatus>(readNumber<std::int32_t>(input)); } - void recurse(const HierarchyObject& hierObj, const DirContainer* oldDirInfo) + void recurse(InSyncDir& container) { - // for (const auto& fileMap : hierObj.refSubFiles()) { processFile(fileMap, oldDirInfo); }); ! + size_t fileCount = readNumber<std::uint32_t>(inputBoth); + while (fileCount-- != 0) //files + { + const Zstring shortName = readUtf8(inputBoth); + const auto inSyncType = static_cast<InSyncFile::InSyncType>(readNumber<std::int32_t>(inputBoth)); + + FileDescriptor dataL; + FileDescriptor dataR; + read(inputLeft, dataL); + read(inputRight, dataR); + + container.addFile(shortName, dataL, dataR, inSyncType); + } + + size_t linkCount = readNumber<std::uint32_t>(inputBoth); + while (linkCount-- != 0) //files + { + const Zstring shortName = readUtf8(inputBoth); + + LinkDescriptor dataL; + LinkDescriptor dataR; + read(inputLeft, dataL); + read(inputRight, dataR); + + container.addSymlink(shortName, dataL, dataR); + } + + size_t dirCount = readNumber<std::uint32_t>(inputBoth); + while (dirCount-- != 0) //files + { + const Zstring shortName = readUtf8(inputBoth); - std::for_each(hierObj.refSubFiles().begin(), hierObj.refSubFiles().end(), [&](const FileMapping& fileMap) { this->processFile(fileMap, oldDirInfo); }); - writePOD<bool>(false); //mark last entry - std::for_each(hierObj.refSubLinks().begin(), hierObj.refSubLinks().end(), [&](const SymLinkMapping& linkObj) { this->processLink(linkObj, oldDirInfo); }); - writePOD<bool>(false); //mark last entry - std::for_each(hierObj.refSubDirs ().begin(), hierObj.refSubDirs ().end(), [&](const DirMapping& dirMap) { this->processDir (dirMap, oldDirInfo); }); - writePOD<bool>(false); //mark last entry + InSyncDir::InSyncStatus status = InSyncDir::STATUS_STRAW_MAN; + read(inputBoth, status); + + InSyncDir& subDir = container.addDir(shortName, status); + recurse(subDir); + } } - void writeStringUtf8(const Zstring& str) { writeString(utfCvrtTo<Zbase<char>>(str)); } + BinStreamIn inputLeft; //data related to one side only + BinStreamIn inputRight; // + BinStreamIn inputBoth; //data concerning both sides +}; - void writeFileId(const FileId& id) +//####################################################################################################################################### + +class UpdateLastSynchronousState +{ + /* + 1. filter by file name does *not* create a new hierarchy, but merely gives a different *view* on the existing file hierarchy + => only update database entries matching this view! + 2. Symlink handling *does* create a new (asymmetric) hierarchies during comparison + => update all database entries! + */ +public: + static void execute(const BaseDirMapping& baseMapping, InSyncDir& dir) { - writePOD<std::uint64_t>(id.first ); //device id - writePOD<std::uint64_t>(id.second); //file id + bool binaryComparison = false; + switch (baseMapping.getCompVariant()) + { + case CMP_BY_TIME_SIZE: + break; + case CMP_BY_CONTENT: + binaryComparison = true; + break; + } + + UpdateLastSynchronousState updater(baseMapping.getFilter(), binaryComparison); + updater.recurse(baseMapping, dir); } -#ifdef _MSC_VER - warn_static("support multiple folder pairs that differ in hard filter only?") -#endif +private: + UpdateLastSynchronousState(const HardFilter& filter, bool binaryComparison) : + filter_(filter), + binaryComparison_(binaryComparison) {} - void processFile(const FileMapping& fileMap, const DirContainer* oldParentDir) + void recurse(const HierarchyObject& hierObj, InSyncDir& dir) { - if (fileMap.getCategory() == FILE_EQUAL) //data in sync: write current state + process(hierObj.refSubFiles(), hierObj.getObjRelativeNamePf(), dir.files); + process(hierObj.refSubLinks(), hierObj.getObjRelativeNamePf(), dir.symlinks); + process(hierObj.refSubDirs (), hierObj.getObjRelativeNamePf(), dir.dirs); + } + + template <class M, class V> + static V& updateItem(M& map, const Zstring& key, const V& value) //efficient create or update without "default-constructible" requirement (Effective STL, item 24) + { + auto iter = map.lower_bound(key); + if (iter != map.end() && !(map.key_comp()(key, iter->first))) { - if (!fileMap.isEmpty<side>()) +#ifdef FFS_WIN //caveat: key might need to be updated, too, if there is a change in short name case!!! + if (iter->first != key) { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(fileMap.getShortName<side>()); //save respecting case! (Windows) - writePOD<std:: int64_t>(to<std:: int64_t>(fileMap.getLastWriteTime<side>())); - writePOD<std::uint64_t>(to<std::uint64_t>(fileMap.getFileSize<side>())); - writeFileId(fileMap.getFileId<side>()); + map.erase(iter); //don't fiddle with decrementing "iter"! - you might lose while optimizing pointlessly + return map.insert(typename M::value_type(key, value)).first->second; } - } - else //not in sync: reuse last synchronous state - { - if (oldParentDir) //no data is also a "synchronous state"! + else +#endif { - auto iter = oldParentDir->files.find(fileMap.getObjShortName()); - if (iter != oldParentDir->files.end()) - { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(iter->first); //save respecting case! (Windows) - writePOD<std:: int64_t>(to<std:: int64_t>(iter->second.lastWriteTimeRaw)); - writePOD<std::uint64_t>(to<std::uint64_t>(iter->second.fileSize)); - writeFileId(iter->second.id); - } + iter->second = value; + return iter->second; } } + return map.insert(iter, typename M::value_type(key, value))->second; } - void processLink(const SymLinkMapping& linkObj, const DirContainer* oldParentDir) + void process(const HierarchyObject::SubFileVec& currentFiles, const Zstring& parentRelativeNamePf, InSyncDir::FileList& dbFiles) { - if (linkObj.getLinkCategory() == SYMLINK_EQUAL) //data in sync: write current state - { - if (!linkObj.isEmpty<side>()) - { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(linkObj.getShortName<side>()); //save respecting case! (Windows) - writePOD<std::int64_t>(to<std::int64_t>(linkObj.getLastWriteTime<side>())); - writeStringUtf8(linkObj.getTargetPath<side>()); - writePOD<std::int32_t>(linkObj.getLinkType<side>()); - } - } - else //not in sync: reuse last synchronous state + hash_set<const InSyncFile*> toPreserve; //referencing fixed-in-memory std::map elements + std::for_each(currentFiles.begin(), currentFiles.end(), [&](const FileMapping& fileMap) { - if (oldParentDir) //no data is also a "synchronous state"! + if (!fileMap.isEmpty()) { - auto iter = oldParentDir->links.find(linkObj.getObjShortName()); - if (iter != oldParentDir->links.end()) + if (fileMap.getCategory() == FILE_EQUAL) //data in sync: write current state { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(iter->first); //save respecting case! (Windows) - writePOD<std::int64_t>(to<std::int64_t>(iter->second.lastWriteTimeRaw)); - writeStringUtf8(iter->second.targetPath); - writePOD<std::int32_t>(iter->second.type); + //create or update new "in-sync" state + InSyncFile& file = updateItem(dbFiles, fileMap.getObjShortName(), + InSyncFile(FileDescriptor(fileMap.getLastWriteTime<LEFT_SIDE>(), + fileMap.getFileSize <LEFT_SIDE>(), + fileMap.getFileId <LEFT_SIDE>()), + FileDescriptor(fileMap.getLastWriteTime<RIGHT_SIDE>(), + fileMap.getFileSize <RIGHT_SIDE>(), + fileMap.getFileId <RIGHT_SIDE>()), + binaryComparison_ ? + InSyncFile::IN_SYNC_BINARY_EQUAL : + InSyncFile::IN_SYNC_ATTRIBUTES_EQUAL)); //efficient add or update (Effective STL, item 24) + //Caveat: If FILE_EQUAL, we *implicitly* assume equal left and right short names matching case: InSyncDir's mapping tables use short name as a key! + //This makes us silently dependent from code in algorithm.h!!! + toPreserve.insert(&file); + } + else //not in sync: preserve last synchronous state + { + auto iter = dbFiles.find(fileMap.getObjShortName()); + if (iter != dbFiles.end()) + toPreserve.insert(&iter->second); } } - } + }); + + //delete removed items (= "in-sync") from database + map_remove_if(dbFiles, [&](const InSyncDir::FileList::value_type& v) -> bool + { + if (toPreserve.find(&v.second) != toPreserve.end()) + return false; + //all items not existing in "currentFiles" have either been deleted meanwhile or been excluded via filter: + const Zstring& shortName = v.first; + return filter_.passFileFilter(parentRelativeNamePf + shortName); + }); } - void processDir(const DirMapping& dirMap, const DirContainer* oldParentDir) + void process(const HierarchyObject::SubLinkVec& currentLinks, const Zstring& parentRelativeNamePf, InSyncDir::LinkList& dbLinks) { - const DirContainer* oldDir = nullptr; - const Zstring* oldDirName = nullptr; - if (oldParentDir) //no data is also a "synchronous state"! + hash_set<const InSyncSymlink*> toPreserve; + std::for_each(currentLinks.begin(), currentLinks.end(), [&](const SymLinkMapping& linkMap) { - auto iter = oldParentDir->dirs.find(dirMap.getObjShortName()); - if (iter != oldParentDir->dirs.end()) + if (!linkMap.isEmpty()) { - oldDirName = &iter->first; - oldDir = &iter->second; + if (linkMap.getLinkCategory() == SYMLINK_EQUAL) //data in sync: write current state + { + //create or update new "in-sync" state + InSyncSymlink& link = updateItem(dbLinks, linkMap.getObjShortName(), + InSyncSymlink(LinkDescriptor(linkMap.getLastWriteTime<LEFT_SIDE>(), + linkMap.getTargetPath <LEFT_SIDE>(), + linkMap.getLinkType <LEFT_SIDE>()), + LinkDescriptor(linkMap.getLastWriteTime<RIGHT_SIDE>(), + linkMap.getTargetPath <RIGHT_SIDE>(), + linkMap.getLinkType <RIGHT_SIDE>()))); //efficient add or update (Effective STL, item 24) + toPreserve.insert(&link); + } + else //not in sync: preserve last synchronous state + { + auto iter = dbLinks.find(linkMap.getObjShortName()); + if (iter != dbLinks.end()) + toPreserve.insert(&iter->second); + } } - } + }); - CompareDirResult cat = dirMap.getDirCategory(); - - if (cat == DIR_EQUAL) //data in sync: write current state + //delete removed items (= "in-sync") from database + map_remove_if(dbLinks, [&](const InSyncDir::LinkList::value_type& v) -> bool { - if (!dirMap.isEmpty<side>()) - { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(dirMap.getShortName<side>()); //save respecting case! (Windows) - recurse(dirMap, oldDir); - } - } - else //not in sync: reuse last synchronous state - { - if (oldDir) - { - writePOD<bool>(true); //mark beginning of entry - writeStringUtf8(*oldDirName); //save respecting case! (Windows) - recurse(dirMap, oldDir); - return; - } - //no data is also a "synchronous state"! + if (toPreserve.find(&v.second) != toPreserve.end()) + return false; + //all items not existing in "currentLinks" have either been deleted meanwhile or been excluded via filter: + const Zstring& shortName = v.first; + return filter_.passFileFilter(parentRelativeNamePf + shortName); + }); + } - //else: not in sync AND no "last synchronous state" - //we cannot simply skip the whole directory, since sub-items might be in sync - //Example: directories on left and right differ in case while sub-files are equal - switch (cat) + void process(const HierarchyObject::SubDirVec& currentDirs, const Zstring& parentRelativeNamePf, InSyncDir::DirList& dbDirs) + { + hash_set<const InSyncDir*> toPreserve; + std::for_each(currentDirs.begin(), currentDirs.end(), [&](const DirMapping& dirMap) + { + if (!dirMap.isEmpty()) { - case DIR_LEFT_SIDE_ONLY: //sub-items cannot be in sync - break; - case DIR_RIGHT_SIDE_ONLY: //sub-items cannot be in sync + switch (dirMap.getDirCategory()) + { + case DIR_EQUAL: + { + //update directory entry only (shallow), but do *not touch* exising child elements!!! + const Zstring& key = dirMap.getObjShortName(); + auto insertResult = dbDirs.insert(std::make_pair(key, InSyncDir(InSyncDir::STATUS_IN_SYNC))); //get or create + auto iter = insertResult.first; + +#ifdef FFS_WIN //caveat: key might need to be updated, too, if there is a change in short name case!!! + const bool alreadyExisting = !insertResult.second; + if (alreadyExisting && iter->first != key) + { + auto oldValue = std::move(iter->second); + dbDirs.erase(iter); //don't fiddle with decrementing "iter"! - you might lose while optimizing pointlessly + iter = dbDirs.insert(InSyncDir::DirList::value_type(key, std::move(oldValue))).first; + } +#endif + InSyncDir& dir = iter->second; + dir.status = InSyncDir::STATUS_IN_SYNC; //update immediate directory entry + toPreserve.insert(&dir); + recurse(dirMap, dir); + } break; - case DIR_EQUAL: - assert(false); + + case DIR_DIFFERENT_METADATA: + //if DIR_DIFFERENT_METADATA and no old database entry yet: we have to insert a new (bogus) database entry: + //we cannot simply skip the whole directory, since sub-items might be in sync! + //Example: directories on left and right differ in case while sub-files are equal + { + //reuse last "in-sync" if available or insert strawman entry (do not try to update thereby removing child elements!!!) + InSyncDir& dir = dbDirs.insert(std::make_pair(dirMap.getObjShortName(), InSyncDir(InSyncDir::STATUS_STRAW_MAN))).first->second; + toPreserve.insert(&dir); + recurse(dirMap, dir); + } break; - case DIR_DIFFERENT_METADATA: - writePOD<bool>(true); - writeStringUtf8(dirMap.getShortName<side>()); - //ATTENTION: strictly this is a violation of the principle of reporting last synchronous state! - //however in this case this will result in "last sync unsuccessful" for this directory within <automatic> algorithm, which is fine - recurse(dirMap, oldDir); //recurse and save sub-items which are in sync + + //not in sync: reuse last synchronous state: + case DIR_LEFT_SIDE_ONLY: + case DIR_RIGHT_SIDE_ONLY: + { + auto iter = dbDirs.find(dirMap.getObjShortName()); + if (iter != dbDirs.end()) + { + toPreserve.insert(&iter->second); + recurse(dirMap, iter->second); //although existing sub-items cannot be in sync, items deleted on both sides *are* in-sync!!! + } + } break; + } } - } + }); + + //delete removed items (= "in-sync") from database + map_remove_if(dbDirs, [&](const InSyncDir::DirList::value_type& v) -> bool + { + if (toPreserve.find(&v.second) != toPreserve.end()) + return false; + //all items not existing in "currentDirs" have either been deleted meanwhile or been excluded via filter: + const Zstring& shortName = v.first; + return filter_.passDirFilter(parentRelativeNamePf + shortName, nullptr); + //if directory is not included in "currentDirs", it is either not existing anymore, in which case it should be deleted from database + //or it was excluded via filter, in which case the database entry should be preserved -> we can't tell and need to preserve the old db entry + }); } + + const HardFilter& filter_; //filter used while scanning directory: generates view on actual files! + const bool binaryComparison_; }; } -//####################################################################################################################################### +//####################################################################################################################################### -std::pair<DirInfoPtr, DirInfoPtr> zen::loadFromDisk(const BaseDirMapping& baseMapping) //throw FileError +std::shared_ptr<InSyncDir> zen::loadLastSynchronousState(const BaseDirMapping& baseMapping) //throw FileError, FileErrorDatabaseNotExisting -> return value always bound! { const Zstring fileNameLeft = getDBFilename<LEFT_SIDE >(baseMapping); const Zstring fileNameRight = getDBFilename<RIGHT_SIDE>(baseMapping); + if (!baseMapping.wasExisting<LEFT_SIDE >() || + !baseMapping.wasExisting<RIGHT_SIDE>()) + { + //avoid race condition with directory existence check: reading sync.ffs_db may succeed although first dir check had failed => conflicts! + //https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3531351&group_id=234430 + const Zstring filename = !baseMapping.wasExisting<LEFT_SIDE>() ? fileNameLeft : fileNameRight; + throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + L" \n" + //it could be due to a to-be-created target directory not yet existing => FileErrorDatabaseNotExisting + replaceCpy(_("Database file %x does not yet exist."), L"%x", fmtFileName(filename))); + } + //read file data: list of session ID + DirInfo-stream - const StreamMapping streamListLeft = ::loadStreams(fileNameLeft); //throw FileError - const StreamMapping streamListRight = ::loadStreams(fileNameRight); //throw FileError + const StreamMapping streamListLeft = ::loadStreams(fileNameLeft); //throw FileError, FileErrorDatabaseNotExisting + const StreamMapping streamListRight = ::loadStreams(fileNameRight); // //find associated session: there can be at most one session within intersection of left and right ids for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft) @@ -430,20 +694,18 @@ std::pair<DirInfoPtr, DirInfoPtr> zen::loadFromDisk(const BaseDirMapping& baseMa auto iterRight = streamListRight.find(iterLeft->first); if (iterRight != streamListRight.end()) { - //read streams into DirInfo - DirInfoPtr dirInfoLeft = StreamParser::execute(iterLeft ->second, fileNameLeft); //throw FileError - DirInfoPtr dirInfoRight = StreamParser::execute(iterRight->second, fileNameRight); //throw FileError - - return std::make_pair(dirInfoLeft, dirInfoRight); + return StreamParser::execute(iterLeft ->second, //throw FileError + iterRight->second, + fileNameLeft, + fileNameRight); } } - throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + L" \n" + _("Database files do not share a common session.")); } -void zen::saveToDisk(const BaseDirMapping& baseMapping) //throw FileError +void zen::saveLastSynchronousState(const BaseDirMapping& baseMapping) //throw FileError { //transactional behaviour! write to tmp files first const Zstring dbNameLeftTmp = getDBFilename<LEFT_SIDE >(baseMapping, true); @@ -468,72 +730,252 @@ void zen::saveToDisk(const BaseDirMapping& baseMapping) //throw FileError //if error occurs: just overwrite old file! User is already informed about issues right after comparing! //find associated session: there can be at most one session within intersection of left and right ids - auto streamLeftOld = streamListLeft .cend(); - auto streamRightOld = streamListRight.cend(); + auto streamIterLeftOld = streamListLeft .cend(); + auto streamIterRightOld = streamListRight.cend(); for (auto iterLeft = streamListLeft.begin(); iterLeft != streamListLeft.end(); ++iterLeft) { auto iterRight = streamListRight.find(iterLeft->first); if (iterRight != streamListRight.end()) { - streamLeftOld = iterLeft; - streamRightOld = iterRight; + streamIterLeftOld = iterLeft; + streamIterRightOld = iterRight; break; } } - //(try to) read old DirInfo - DirInfoPtr dirInfoLeftOld; - DirInfoPtr dirInfoRightOld; - if (streamLeftOld != streamListLeft .end() && - streamRightOld != streamListRight.end()) + //load last synchrounous state + std::shared_ptr<InSyncDir> lastSyncState = std::make_shared<InSyncDir>(InSyncDir::STATUS_IN_SYNC); + if (streamIterLeftOld != streamListLeft .end() && + streamIterRightOld != streamListRight.end()) try { - dirInfoLeftOld = StreamParser::execute(streamLeftOld ->second, dbNameLeft ); //throw FileError - dirInfoRightOld = StreamParser::execute(streamRightOld->second, dbNameRight); //throw FileError - } - catch (FileError&) - { - //if error occurs: just overwrite old file! User is already informed about issues right after comparing! - dirInfoLeftOld .reset(); //read both or none! - dirInfoRightOld.reset(); // + lastSyncState = StreamParser::execute(streamIterLeftOld ->second, //throw FileError + streamIterRightOld->second, + dbNameLeft, + dbNameRight); } + catch (FileError&) {} //if error occurs: just overwrite old file! User is already informed about issues right after comparing! - //create new database entries - MemoryStream rawStreamLeftNew = StreamGenerator<LEFT_SIDE >::execute(baseMapping, dirInfoLeftOld .get() ? &dirInfoLeftOld ->baseDirContainer : nullptr, dbNameLeft); - MemoryStream rawStreamRightNew = StreamGenerator<RIGHT_SIDE>::execute(baseMapping, dirInfoRightOld.get() ? &dirInfoRightOld->baseDirContainer : nullptr, dbNameRight); + //update last synchrounous state + UpdateLastSynchronousState::execute(baseMapping, *lastSyncState); + + //serialize again + BinaryStream updatedStreamLeft; + BinaryStream updatedStreamRight; + StreamGenerator::execute(*lastSyncState, + dbNameLeft, + dbNameRight, + updatedStreamLeft, + updatedStreamRight, false); //throw FileError //check if there is some work to do at all - if (streamLeftOld != streamListLeft .end() && rawStreamLeftNew == streamLeftOld ->second && - streamRightOld != streamListRight.end() && rawStreamRightNew == streamRightOld->second) + if (streamIterLeftOld != streamListLeft .end() && updatedStreamLeft == streamIterLeftOld ->second && + streamIterRightOld != streamListRight.end() && updatedStreamRight == streamIterRightOld->second) return; //some users monitor the *.ffs_db file with RTS => don't touch the file if it isnt't strictly needed //erase old session data - if (streamLeftOld != streamListLeft.end()) - streamListLeft.erase(streamLeftOld); - if (streamRightOld != streamListRight.end()) - streamListRight.erase(streamRightOld); + if (streamIterLeftOld != streamListLeft.end()) + streamListLeft.erase(streamIterLeftOld); + if (streamIterRightOld != streamListRight.end()) + streamListRight.erase(streamIterRightOld); //create/update DirInfo-streams const std::string sessionID = zen::generateGUID(); //fill in new - streamListLeft .insert(std::make_pair(sessionID, rawStreamLeftNew)); - streamListRight.insert(std::make_pair(sessionID, rawStreamRightNew)); + streamListLeft [sessionID] = std::move(updatedStreamLeft); + streamListRight[sessionID] = std::move(updatedStreamRight); //write (temp-) files... zen::ScopeGuard guardTempFileLeft = zen::makeGuard([&] {zen::removeFile(dbNameLeftTmp); }); - saveFile(streamListLeft, dbNameLeftTmp); //throw FileError + saveStreams(streamListLeft, dbNameLeftTmp); //throw FileError zen::ScopeGuard guardTempFileRight = zen::makeGuard([&] {zen::removeFile(dbNameRightTmp); }); - saveFile(streamListRight, dbNameRightTmp); //throw FileError + saveStreams(streamListRight, dbNameRightTmp); //throw FileError //operation finished: rename temp files -> this should work transactionally: //if there were no write access, creation of temp files would have failed - removeFile(dbNameLeft); - removeFile(dbNameRight); - renameFile(dbNameLeftTmp, dbNameLeft); //throw FileError; - renameFile(dbNameRightTmp, dbNameRight); //throw FileError; + removeFile(dbNameLeft); // + removeFile(dbNameRight); //throw FileError + renameFile(dbNameLeftTmp, dbNameLeft); // + renameFile(dbNameRightTmp, dbNameRight); // guardTempFileLeft. dismiss(); //no need to delete temp file anymore guardTempFileRight.dismiss(); // } + +#ifdef FFS_WIN +warn_static("remove after migration") +#endif + +namespace +{ +class CheckedDbReader : public CheckedReader +{ +public: + CheckedDbReader(wxInputStream& stream, const Zstring& errorObjName) : CheckedReader(stream), errorObjName_(errorObjName) {} + +private: + virtual void throwException() const { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(errorObjName_))); } + + const Zstring errorObjName_; +}; + + +class StreamParser_v8 //for db-file back-wards compatibility we stick with two output streams until further +{ +public: + static std::shared_ptr<InSyncDir> execute(const BinaryStream& streamL, const BinaryStream& streamR, //throw FileError + const Zstring& filenameL, //used for diagnostics only + const Zstring& filenameR) + { + try + { + auto output = std::make_shared<InSyncDir>(InSyncDir::STATUS_IN_SYNC); + StreamParser_v8 parser(streamL, streamR); //throw UnexpectedEndOfStreamError, std::bad_alloc + parser.recurse(*output); + return output; + } + catch (const UnexpectedEndOfStreamError&) + { + throw FileError(_("Database file is corrupt:") + L"\n" + fmtFileName(filenameL) + L"\n" + fmtFileName(filenameR)); + } + catch (const std::bad_alloc& e) + { + throw FileError(_("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); + } + } + +private: + StreamParser_v8(const BinaryStream& bufferL, + const BinaryStream& bufferR) : + inputLeft (bufferL), //input is referenced only! + inputRight(bufferR) {} + + static Zstring readUtf8(BinStreamIn& input) { return utfCvrtTo<Zstring>(readContainer<Zbase<char>>(input)); } //throw UnexpectedEndOfStreamError + + static void read(BinStreamIn& input, Zstring& shortName, FileDescriptor& descr) + { + //attention: order of function argument evaluation is undefined! So do it one after the other... + shortName = readUtf8(input); + descr.lastWriteTimeRaw = readNumber<std::int64_t>(input); //throw UnexpectedEndOfStreamError + descr.fileSize = readNumber<std::uint64_t>(input); + descr.id.first = static_cast<decltype(descr.id.first )>(readNumber<std::uint64_t>(input)); // + descr.id.second = static_cast<decltype(descr.id.second)>(readNumber<std::uint64_t>(input)); //silence "loss of precision" compiler warnings + } + + static void read(BinStreamIn& input, Zstring& shortName, LinkDescriptor& descr) + { + shortName = readUtf8(input); + descr.lastWriteTimeRaw = readNumber<std::int64_t>(input); + descr.targetPath = readUtf8(input); //file name + descr.type = static_cast<LinkDescriptor::LinkType>(readNumber<std::int32_t>(input)); + } + + void recurse(InSyncDir& dir) + { + for (;;) //files + { + bool haveItemL = readNumber<bool>(inputLeft ); //remove redundancy in next db format + bool haveItemR = readNumber<bool>(inputRight); // + assert(haveItemL == haveItemR); + if (!haveItemL || !haveItemR) break; + + Zstring shortName; + FileDescriptor dataL; + FileDescriptor dataR; + read(inputLeft, shortName, dataL); + read(inputRight, shortName, dataR); + + dir.addFile(shortName, dataL, dataR, InSyncFile::IN_SYNC_ATTRIBUTES_EQUAL); + } + + for (;;) //symlinks + { + bool haveItemL = readNumber<bool>(inputLeft ); + bool haveItemR = readNumber<bool>(inputRight); + assert(haveItemL == haveItemR); + if (!haveItemL || !haveItemR) break; + + Zstring shortName; + LinkDescriptor dataL; + LinkDescriptor dataR; + read(inputLeft, shortName, dataL); + read(inputRight, shortName, dataR); + + dir.addSymlink(shortName, dataL, dataR); + } + + for (;;) //directories + { + bool haveItemL = readNumber<bool>(inputLeft ); + bool haveItemR = readNumber<bool>(inputRight); + assert(haveItemL == haveItemR); + if (!haveItemL || !haveItemR) break; + + Zstring shortName = readUtf8(inputLeft); + shortName = readUtf8(inputRight); + InSyncDir& subDir = dir.addDir(shortName, InSyncDir::STATUS_IN_SYNC); + recurse(subDir); + } + } + + BinStreamIn inputLeft; + BinStreamIn inputRight; +}; + + +StreamMapping loadStreams_v8(const Zstring& filename) //throw FileError +{ + try + { + //read format description (uncompressed) + FileInputStream rawStream(filename); //throw FileError, ErrorNotExisting + + //read FreeFileSync file identifier + char formatDescr[sizeof(FILE_FORMAT_DESCR)] = {}; + rawStream.Read(formatDescr, sizeof(formatDescr)); //throw FileError + + if (!std::equal(FILE_FORMAT_DESCR, FILE_FORMAT_DESCR + sizeof(FILE_FORMAT_DESCR), formatDescr)) + throw FileError(replaceCpy(_("Database file %x is incompatible."), L"%x", fmtFileName(filename))); + + wxZlibInputStream decompressed(rawStream, wxZLIB_ZLIB); + + CheckedDbReader cr(decompressed, filename); + + std::int32_t version = cr.readPOD<std::int32_t>(); + if (version != 8) //read file format version# + throw FileError(replaceCpy(_("Database file %x is incompatible."), L"%x", fmtFileName(filename))); + + //read stream lists + StreamMapping output; + + std::uint32_t dbCount = cr.readPOD<std::uint32_t>(); //number of databases: one for each sync-pair + while (dbCount-- != 0) + { + //DB id of partner databases + std::string sessionID = cr.readString<std::string>(); + BinaryStream stream = cr.readString<BinaryStream>(); //read db-entry stream (containing DirInformation) + + //convert streams + std::shared_ptr<InSyncDir> lastSyncState = StreamParser_v8::execute(stream, stream, filename, filename); //throw FileError + + //serialize again + BinaryStream strL; + BinaryStream strR; + StreamGenerator::execute(*lastSyncState, filename, filename, strL, strR, true); //throw FileError + output[sessionID] = std::move(strL); + } + return output; + } + catch (ErrorNotExisting&) + { + throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + L" \n" + + replaceCpy(_("Database file %x does not yet exist."), L"%x", fmtFileName(filename))); + } + catch (const std::bad_alloc& e) + { + throw FileError(_("Out of memory!") + L" " + utfCvrtTo<std::wstring>(e.what())); + } +} +} diff --git a/lib/db_file.h b/lib/db_file.h index e5ae7b50..4425f52c 100644 --- a/lib/db_file.h +++ b/lib/db_file.h @@ -14,17 +14,75 @@ namespace zen { const Zstring SYNC_DB_FILE_ENDING = Zstr(".ffs_db"); -struct DirInformation +//artificial hierarchy of last synchronous state: +struct InSyncFile { - DirContainer baseDirContainer; //hierarchical directory information + enum InSyncType + { + IN_SYNC_BINARY_EQUAL, //checked file content + IN_SYNC_ATTRIBUTES_EQUAL, //only "looks" like they're equal + }; + + InSyncFile(const FileDescriptor& l, const FileDescriptor& r, InSyncType type) : left(l), right(r), inSyncType(type) {} + FileDescriptor left; + FileDescriptor right; + InSyncType inSyncType; +}; + +struct InSyncSymlink +{ + InSyncSymlink(const LinkDescriptor& l, const LinkDescriptor& r) : left(l), right(r) {} + LinkDescriptor left; + LinkDescriptor right; +}; + +struct InSyncDir +{ + //for directories we have a logical problem: we cannot have "not existent" as an indicator for "no last synchronous state" since this precludes + //child elements that may be in sync! + enum InSyncStatus + { + STATUS_IN_SYNC, + STATUS_STRAW_MAN //there is no last synchronous state, but used as container only + }; + InSyncDir(InSyncStatus statusIn) : status(statusIn) {} + + InSyncStatus status; + + //------------------------------------------------------------------ + typedef std::map<Zstring, InSyncDir, LessFilename> DirList; // + typedef std::map<Zstring, InSyncFile, LessFilename> FileList; // key: shortName + typedef std::map<Zstring, InSyncSymlink, LessFilename> LinkList; // + //------------------------------------------------------------------ + + DirList dirs; + FileList files; + LinkList symlinks; //non-followed symlinks + + //convenience + InSyncDir& addDir(const Zstring& shortName, InSyncStatus statusIn) + { + //use C++11 emplace when available + return dirs.insert(std::make_pair(shortName, InSyncDir(statusIn))).first->second; + } + + void addFile(const Zstring& shortName, const FileDescriptor& dataL, const FileDescriptor& dataR, InSyncFile::InSyncType type) + { + files.insert(std::make_pair(shortName, InSyncFile(dataL, dataR, type))); + } + + void addSymlink(const Zstring& shortName, const LinkDescriptor& dataL, const LinkDescriptor& dataR) + { + symlinks.insert(std::make_pair(shortName, InSyncSymlink(dataL, dataR))); + } }; -typedef std::shared_ptr<const DirInformation> DirInfoPtr; + DEFINE_NEW_FILE_ERROR(FileErrorDatabaseNotExisting); -std::pair<DirInfoPtr, DirInfoPtr> loadFromDisk(const BaseDirMapping& baseMapping); //throw FileError, FileErrorDatabaseNotExisting -> return value always bound! +std::shared_ptr<InSyncDir> loadLastSynchronousState(const BaseDirMapping& baseMapping); //throw FileError, FileErrorDatabaseNotExisting -> return value always bound! -void saveToDisk(const BaseDirMapping& baseMapping); //throw FileError +void saveLastSynchronousState(const BaseDirMapping& baseMapping); //throw FileError } #endif // DBFILE_H_INCLUDED diff --git a/lib/dir_lock.cpp b/lib/dir_lock.cpp index f41bbfa8..682612a7 100644 --- a/lib/dir_lock.cpp +++ b/lib/dir_lock.cpp @@ -9,17 +9,15 @@ #include <wx/log.h> //#include <wx/msgdlg.h> #include <memory> -#include <wx+/string_conv.h> #include <zen/last_error.h> #include <zen/thread.h> //includes <boost/thread.hpp> #include <zen/scope_guard.h> #include <zen/guid.h> -#include <zen/file_io.h> #include <zen/tick_count.h> #include <zen/assert_static.h> -#include <wx+/serialize.h> #include <zen/int64.h> #include <zen/file_handling.h> +#include <wx+/serialize.h> #ifdef FFS_WIN #include <tlhelp32.h> @@ -161,27 +159,6 @@ Zstring deleteAbandonedLockName(const Zstring& lockfilename) //make sure to NOT } -class CheckedLockReader : public CheckedReader -{ -public: - CheckedLockReader(wxInputStream& stream, const Zstring& errorObjName) : CheckedReader(stream), errorObjName_(errorObjName) {} - virtual void throwException() const { throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(errorObjName_))); } - -private: - const Zstring errorObjName_; -}; - -class CheckedLockWriter : public CheckedWriter -{ -public: - CheckedLockWriter(wxOutputStream& stream, const Zstring& errorObjName) : CheckedWriter(stream), errorObjName_(errorObjName) {} - virtual void throwException() const { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtFileName(errorObjName_))); } - -private: - const Zstring errorObjName_; -}; - - #ifdef FFS_WIN std::wstring getLoginSid() //throw FileError { @@ -273,35 +250,35 @@ struct LockInformation //throw FileError } #endif - explicit LockInformation(CheckedLockReader& reader) + explicit LockInformation(BinStreamIn& stream) //throw UnexpectedEndOfStreamError { char tmp[sizeof(LOCK_FORMAT_DESCR)] = {}; - reader.readArray(&tmp, sizeof(tmp)); //file format header - const int lockFileVersion = reader.readPOD<boost::int32_t>(); // + readArray(stream, &tmp, sizeof(tmp)); //file format header + const int lockFileVersion = readNumber<boost::int32_t>(stream); // if (!std::equal(std::begin(tmp), std::end(tmp), std::begin(LOCK_FORMAT_DESCR)) || lockFileVersion != LOCK_FORMAT_VER) - reader.throwException(); + throw UnexpectedEndOfStreamError(); //well, not really...!? - reader.readString(lockId); - reader.readString(computerName); - reader.readString(userId); - reader.readString(sessionId); - processId = static_cast<decltype(processId)>(reader.readPOD<std::uint64_t>()); //[!] conversion + lockId = readContainer<std::string>(stream); // + computerName = readContainer<std::string>(stream); //UnexpectedEndOfStreamError + userId = readContainer<std::string>(stream); // + sessionId = readContainer<std::string>(stream); // + processId = static_cast<decltype(processId)>(readNumber<std::uint64_t>(stream)); //[!] conversion } - void toStream(CheckedLockWriter& writer) const + void toStream(BinStreamOut& stream) const //throw () { - writer.writeArray(LOCK_FORMAT_DESCR, sizeof(LOCK_FORMAT_DESCR)); - writer.writePOD<boost::int32_t>(LOCK_FORMAT_VER); + writeArray(stream, LOCK_FORMAT_DESCR, sizeof(LOCK_FORMAT_DESCR)); + writeNumber<boost::int32_t>(stream, LOCK_FORMAT_VER); assert_static(sizeof(processId) <= sizeof(std::uint64_t)); //ensure portability - writer.writeString(lockId); - writer.writeString(computerName); - writer.writeString(userId); - writer.writeString(sessionId); - writer.writePOD<std::uint64_t>(processId); + writeContainer(stream, lockId); + writeContainer(stream, computerName); + writeContainer(stream, userId); + writeContainer(stream, sessionId); + writeNumber<std::uint64_t>(stream, processId); } std::string lockId; //16 byte GUID - a universal identifier for this lock (no matter what the path is, considering symlinks, distributed network, etc.) @@ -322,17 +299,23 @@ struct LockInformation //throw FileError void writeLockInfo(const Zstring& lockfilename) //throw FileError { - FileOutputStream stream(lockfilename); //throw FileError - CheckedLockWriter writer(stream, lockfilename); - LockInformation(FromCurrentProcess()).toStream(writer); //throw FileError + BinStreamOut streamOut; + LockInformation(FromCurrentProcess()).toStream(streamOut); + saveBinStream(lockfilename, streamOut.get()); //throw FileError } LockInformation retrieveLockInfo(const Zstring& lockfilename) //throw FileError, ErrorNotExisting { - FileInputStream stream(lockfilename); //throw FileError, ErrorNotExisting - CheckedLockReader reader(stream, lockfilename); - return LockInformation(reader); //throw FileError + BinStreamIn streamIn = loadBinStream<BinaryStream>(lockfilename); //throw FileError, ErrorNotExisting + try + { + return LockInformation(streamIn); //throw UnexpectedEndOfStreamError + } + catch (UnexpectedEndOfStreamError&) + { + throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtFileName(lockfilename))); + } } diff --git a/lib/ffs_paths.h b/lib/ffs_paths.h index 9376825b..d7987195 100644 --- a/lib/ffs_paths.h +++ b/lib/ffs_paths.h @@ -42,7 +42,7 @@ namespace impl inline const Zstring& getBinaryDir() //directory containing executable WITH path separator at end { - static Zstring instance = beforeLast(toZ(wxStandardPaths::Get().GetExecutablePath()), FILE_NAME_SEPARATOR) + Zstring(FILE_NAME_SEPARATOR); //extern linkage! + static Zstring instance = beforeLast(utfCvrtTo<Zstring>(wxStandardPaths::Get().GetExecutablePath()), FILE_NAME_SEPARATOR) + Zstring(FILE_NAME_SEPARATOR); //extern linkage! return instance; } @@ -102,7 +102,7 @@ Zstring getConfigDir() if (!dirExists(userDirectory)) try { - createDirectory(userDirectory); //only top directory needs to be created: no recursion necessary + makeDirectory(userDirectory); //only top directory needs to be created: no recursion necessary } catch (const FileError&) {} diff --git a/lib/generate_logfile.h b/lib/generate_logfile.h new file mode 100644 index 00000000..8feb696a --- /dev/null +++ b/lib/generate_logfile.h @@ -0,0 +1,138 @@ +// ************************************************************************** +// * This file is part of the FreeFileSync project. It is distributed under * +// * GNU General Public License: http://www.gnu.org/licenses/gpl.html * +// * Copyright (C) ZenJu (zhnmju123 AT gmx DOT de) - All Rights Reserved * +// ************************************************************************** + +#ifndef GEN_LOGFILE_H_93172643216748973216458732165415 +#define GEN_LOGFILE_H_93172643216748973216458732165415 + +#include <zen/error_log.h> +#include <zen/file_io.h> +#include <wx+/format_unit.h> +#include <wx+/serialize.h> +#include "ffs_paths.h" + + +namespace zen +{ +Utf8String generateLogStream(const ErrorLog& log, + const std::wstring& jobName, //may be empty + const std::wstring& finalStatus, + int itemsSynced, Int64 dataSynced, + int itemsTotal, Int64 dataTotal, + long totalTime); //unit: [sec] + +void saveToLastSyncsLog(const Utf8String& logstream); //throw FileError + + + + + + + +//####################### implementation ####################### +namespace +{ +Utf8String generateLogStream_impl(const ErrorLog& log, + const std::wstring& jobName, //may be empty + const std::wstring& finalStatus, + int itemsSynced, Int64 dataSynced, + int itemsTotal, Int64 dataTotal, + long totalTime) //unit: [sec] +{ + Utf8String output; + + //write header + std::wstring headerLine = _("Batch execution") + L" - " + formatTime<std::wstring>(FORMAT_DATE); + if (!jobName.empty()) + headerLine += L" - " + jobName; + + //output += utfCvrtTo<MemoryStream>(headerLine); + //output += '\n'; + + //for (size_t i = 0; i < headerLine.size(); ++i) //well, this considers UTF-16 only, not true Unicode... + // output += '='; + //output += '\n'; + + //assemble results box + std::vector<std::wstring> results; + results.push_back(headerLine); + results.push_back(L""); + results.push_back(finalStatus); + results.push_back(L""); + if (itemsTotal != 0 || dataTotal != 0) //=: sync phase was reached and there were actual items to sync + { + results.push_back(L" " + _("Items processed:") + L" " + toGuiString(itemsSynced) + L" (" + filesizeToShortString(dataSynced) + L")"); + + if (itemsSynced != itemsTotal || + dataSynced != dataTotal) + results.push_back(L" " + _("Items remaining:") + L" " + toGuiString(itemsTotal - itemsSynced) + L" (" + filesizeToShortString(dataTotal - dataSynced) + L")"); + } + results.push_back(L" " + _("Total time:") + L" " + copyStringTo<std::wstring>(wxTimeSpan::Seconds(totalTime).Format())); + + //calculate max width, this considers UTF-16 only, not true Unicode... + size_t sepLineLen = 0; + std::for_each(results.begin(), results.end(), [&](const std::wstring& str) { sepLineLen = std::max(sepLineLen, str.size()); }); + + for (size_t i = 0; i < sepLineLen; ++i) output += '_'; + output += "\n"; + + std::for_each(results.begin(), results.end(), [&](const std::wstring& str) { output += utfCvrtTo<Utf8String>(str); output += '\n'; }); + + for (size_t i = 0; i < sepLineLen; ++i) output += '_'; + output += "\n\n"; + + //write log items + const auto& entries = log.getEntries(); + for (auto iter = entries.begin(); iter != entries.end(); ++iter) + { + output += utfCvrtTo<Utf8String>(formatMessage(*iter)); + output += '\n'; + } + + return replaceCpy(output, '\n', LINE_BREAK); //don't replace line break any earlier +} +} + + +inline +Utf8String generateLogStream(const ErrorLog& log, + const std::wstring& jobName, //may be empty + const std::wstring& finalStatus, + int itemsSynced, Int64 dataSynced, + int itemsTotal, Int64 dataTotal, + long totalTime) //unit: [sec] +{ + return generateLogStream_impl(log, jobName, finalStatus, itemsSynced, dataSynced, itemsTotal, dataTotal, totalTime); +} + + +inline +void saveToLastSyncsLog(const Utf8String& logstream) //throw FileError +{ + const Zstring filename = getConfigDir() + Zstr("LastSyncs.log"); + + Utf8String oldStream; + try + { + oldStream = loadBinStream<Utf8String>(filename); //throw FileError, ErrorNotExisting + } + catch (const ErrorNotExisting&) {} + + Utf8String newStream = logstream; + if (!oldStream.empty()) + { + newStream += LINE_BREAK; + newStream += LINE_BREAK; + newStream += oldStream; + } + + //limit file size: 128 kB (but do not truncate new log) + newStream.resize(std::min(newStream.size(), std::max<size_t>(logstream.size(), 128 * 1024))); + + saveBinStream(filename, newStream); //throw FileError +} +} + +#endif //GEN_LOGFILE_H_93172643216748973216458732165415 diff --git a/lib/hard_filter.cpp b/lib/hard_filter.cpp index 603d27b0..d66fc665 100644 --- a/lib/hard_filter.cpp +++ b/lib/hard_filter.cpp @@ -5,14 +5,12 @@ // ************************************************************************** #include "hard_filter.h" -#include <zen/zstring.h> -#include <wx/string.h> #include <set> #include <stdexcept> #include <vector> -#include "../structures.h" -#include <wx+/serialize.h> #include <typeinfo> +#include <iterator> +//#include "../structures.h" using namespace zen; @@ -30,31 +28,31 @@ bool zen::operator<(const HardFilter& lhs, const HardFilter& rhs) } -void HardFilter::saveFilter(wxOutputStream& stream) const //serialize derived object -{ - //save type information - writeString(stream, uniqueClassIdentifier()); - - //save actual object - save(stream); -} - - -HardFilter::FilterRef HardFilter::loadFilter(wxInputStream& stream) -{ - //read type information - const Zstring uniqueClassId = readString<Zstring>(stream); - - //read actual object - if (uniqueClassId == Zstr("NullFilter")) - return NullFilter::load(stream); - else if (uniqueClassId == Zstr("NameFilter")) - return NameFilter::load(stream); - else if (uniqueClassId == Zstr("CombinedFilter")) - return CombinedFilter::load(stream); - else - throw std::logic_error("Programming Error: Unknown filter!"); -} +//void HardFilter::saveFilter(ZstreamOut& stream) const //serialize derived object +//{ +// //save type information +// writeString(stream, uniqueClassIdentifier()); +// +// //save actual object +// save(stream); +//} + + +//HardFilter::FilterRef HardFilter::loadFilter(ZstreamIn& stream) //throw UnexpectedEndOfStreamError +//{ +// //read type information +// const std::string uniqueClassId = readString<std::string>(stream); //throw UnexpectedEndOfStreamError +// +// //read actual object +// if (uniqueClassId == "NullFilter") +// return NullFilter::load(stream); +// else if (uniqueClassId == "NameFilter") +// return NameFilter::load(stream); +// else if (uniqueClassId == "CombinedFilter") +// return CombinedFilter::load(stream); +// else +// throw std::logic_error("Programming Error: Unknown filter!"); +//} namespace @@ -240,7 +238,7 @@ std::vector<Zstring> splitByDelimiter(const Zstring& filterString) //delimiters may be ';' or '\n' std::vector<Zstring> output; - const std::vector<Zstring> blocks = split(filterString, Zchar(';')); + const std::vector<Zstring> blocks = split(filterString, Zchar(';')); //split by less common delimiter first std::for_each(blocks.begin(), blocks.end(), [&](const Zstring& item) { @@ -377,23 +375,17 @@ bool NameFilter::cmpLessSameType(const HardFilter& other) const } -Zstring NameFilter::uniqueClassIdentifier() const -{ - return Zstr("NameFilter"); -} - - -void NameFilter::save(wxOutputStream& stream) const -{ - writeString(stream, includeFilterTmp); - writeString(stream, excludeFilterTmp); -} - - -HardFilter::FilterRef NameFilter::load(wxInputStream& stream) //"constructor" -{ - const Zstring include = readString<Zstring>(stream); - const Zstring exclude = readString<Zstring>(stream); - - return FilterRef(new NameFilter(include, exclude)); -} +//void NameFilter::save(ZstreamOut& stream) const +//{ +// writeString(stream, includeFilterTmp); +// writeString(stream, excludeFilterTmp); +//} +// +// +//HardFilter::FilterRef NameFilter::load(ZstreamIn& stream) //throw UnexpectedEndOfStreamError +//{ +// const Zstring include = readString<Zstring>(stream); //throw UnexpectedEndOfStreamError +// const Zstring exclude = readString<Zstring>(stream); // +// +// return FilterRef(new NameFilter(include, exclude)); +//} diff --git a/lib/hard_filter.h b/lib/hard_filter.h index 1a9943a3..90cd33fc 100644 --- a/lib/hard_filter.h +++ b/lib/hard_filter.h @@ -9,15 +9,15 @@ #include <vector> #include <memory> -#include <wx/stream.h> #include <zen/zstring.h> +//#include <wx+/serialize.h> namespace zen { //------------------------------------------------------------------ /* Semantics of HardFilter: -1. using it creates a NEW folder hierarchy! -> must be considered by <Automatic>-mode! (fortunately it turns out, doing nothing already has perfect semantics :) +1. using it creates a NEW folder hierarchy! -> must be considered by <Automatic>-mode! 2. it applies equally to both sides => it always matches either both sides or none! => can be used while traversing a single folder! class hierarchy: @@ -45,14 +45,14 @@ public: typedef std::shared_ptr<const HardFilter> FilterRef; //always bound by design! //serialization - void saveFilter(wxOutputStream& stream) const; //serialize derived object - static FilterRef loadFilter(wxInputStream& stream); //CAVEAT!!! adapt this method for each new derivation!!! + // void saveFilter(ZstreamOut& stream) const; //serialize derived object + // static FilterRef loadFilter(ZstreamIn& stream); //throw UnexpectedEndOfStreamError; CAVEAT!!! adapt this method for each new derivation!!! private: - friend bool operator< (const HardFilter& lhs, const HardFilter& rhs); + friend bool operator<(const HardFilter& lhs, const HardFilter& rhs); - virtual void save(wxOutputStream& stream) const = 0; //serialization - virtual Zstring uniqueClassIdentifier() const = 0; //get identifier, used for serialization + // virtual void save(ZstreamOut& stream) const = 0; //serialization + virtual std::string uniqueClassIdentifier() const = 0; //get identifier, used for serialization virtual bool cmpLessSameType(const HardFilter& other) const = 0; //typeid(*this) == typeid(other) in this context! }; @@ -74,9 +74,9 @@ public: private: friend class HardFilter; - virtual void save(wxOutputStream& stream) const {} - virtual Zstring uniqueClassIdentifier() const; - static FilterRef load(wxInputStream& stream); //"serial constructor" + // virtual void save(ZstreamOut& stream) const {} + virtual std::string uniqueClassIdentifier() const { return "NullFilter"; } + // static FilterRef load(ZstreamIn& stream); //throw UnexpectedEndOfStreamError virtual bool cmpLessSameType(const HardFilter& other) const; }; @@ -94,9 +94,9 @@ public: private: friend class HardFilter; - virtual void save(wxOutputStream& stream) const; - virtual Zstring uniqueClassIdentifier() const; - static FilterRef load(wxInputStream& stream); //"serial constructor" + // virtual void save(ZstreamOut& stream) const; + virtual std::string uniqueClassIdentifier() const { return "NameFilter"; } + // static FilterRef load(ZstreamIn& stream); //throw UnexpectedEndOfStreamError virtual bool cmpLessSameType(const HardFilter& other) const; std::vector<Zstring> filterFileIn; // @@ -120,9 +120,9 @@ public: private: friend class HardFilter; - virtual void save(wxOutputStream& stream) const; - virtual Zstring uniqueClassIdentifier() const; - static FilterRef load(wxInputStream& stream); //"serial constructor" + // virtual void save(ZstreamOut& stream) const; + virtual std::string uniqueClassIdentifier() const { return "CombinedFilter"; } + // static FilterRef load(ZstreamIn& stream); //throw UnexpectedEndOfStreamError virtual bool cmpLessSameType(const HardFilter& other) const; const FilterRef first_; @@ -147,11 +147,11 @@ private: //---------------Inline Implementation--------------------------------------------------- -inline -HardFilter::FilterRef NullFilter::load(wxInputStream& stream) //"serial constructor" -{ - return FilterRef(new NullFilter); -} +//inline +//HardFilter::FilterRef NullFilter::load(ZstreamIn& stream) +//{ +// return FilterRef(new NullFilter); +//} inline @@ -185,16 +185,9 @@ bool NullFilter::cmpLessSameType(const HardFilter& other) const inline -Zstring NullFilter::uniqueClassIdentifier() const -{ - return Zstr("NullFilter"); -} - - -inline bool CombinedFilter::passFileFilter(const Zstring& relFilename) const { - return first_->passFileFilter(relFilename) && //short-circuit behavior + return first_ ->passFileFilter(relFilename) && //short-circuit behavior second_->passFileFilter(relFilename); } @@ -202,8 +195,14 @@ bool CombinedFilter::passFileFilter(const Zstring& relFilename) const inline bool CombinedFilter::passDirFilter(const Zstring& relDirname, bool* subObjMightMatch) const { - return first_->passDirFilter(relDirname, subObjMightMatch) && //short-circuit behavior: subObjMightMatch handled correctly! - second_->passDirFilter(relDirname, subObjMightMatch); + if (first_->passDirFilter(relDirname, subObjMightMatch)) + return second_->passDirFilter(relDirname, subObjMightMatch); + else + { + if (subObjMightMatch && *subObjMightMatch) + second_->passDirFilter(relDirname, subObjMightMatch); + return false; + } } @@ -228,29 +227,22 @@ bool CombinedFilter::cmpLessSameType(const HardFilter& other) const } -inline -Zstring CombinedFilter::uniqueClassIdentifier() const -{ - return Zstr("CombinedFilter"); -} - - -inline -void CombinedFilter::save(wxOutputStream& stream) const -{ - first_->saveFilter(stream); - second_->saveFilter(stream); -} +//inline +//void CombinedFilter::save(ZstreamOut& stream) const +//{ +// first_ ->saveFilter(stream); +// second_->saveFilter(stream); +//} -inline -HardFilter::FilterRef CombinedFilter::load(wxInputStream& stream) //"constructor" -{ - FilterRef first = loadFilter(stream); - FilterRef second = loadFilter(stream); - - return combineFilters(first, second); -} +//inline +//HardFilter::FilterRef CombinedFilter::load(ZstreamIn& stream) //throw UnexpectedEndOfStreamError +//{ +// FilterRef first = loadFilter(stream); //throw UnexpectedEndOfStreamError +// FilterRef second = loadFilter(stream); // +// +// return combineFilters(first, second); +//} inline @@ -272,8 +264,6 @@ HardFilter::FilterRef combineFilters(const HardFilter::FilterRef& first, return HardFilter::FilterRef(new CombinedFilter(first, second)); } } - - } diff --git a/lib/icon_buffer.cpp b/lib/icon_buffer.cpp index daf32a86..2dc6f389 100644 --- a/lib/icon_buffer.cpp +++ b/lib/icon_buffer.cpp @@ -9,7 +9,6 @@ #include <set> #include <zen/thread.h> //includes <boost/thread.hpp> #include <zen/scope_guard.h> -//#include <boost/thread/once.hpp> #ifdef FFS_WIN #include <zen/dll.h> @@ -27,7 +26,7 @@ namespace { const size_t BUFFER_SIZE_MAX = 800; //maximum number of icons to buffer - +inline int cvrtSize(IconBuffer::IconSize sz) //get size in pixel { switch (sz) @@ -57,7 +56,7 @@ public: typedef GdkPixbuf* HandleType; #endif - explicit IconHolder(HandleType handle = 0) : handle_(handle) {} //take ownership! + explicit IconHolder(HandleType handle = nullptr) : handle_(handle) {} //take ownership! //icon holder has value semantics! IconHolder(const IconHolder& other) : handle_(other.handle_ == nullptr ? nullptr : @@ -104,31 +103,36 @@ public: ICONINFO icoInfo = {}; if (::GetIconInfo(clone.handle_, &icoInfo)) { - ::DeleteObject(icoInfo.hbmMask); //nice potential for a GDI leak! - ZEN_ON_SCOPE_EXIT(::DeleteObject(icoInfo.hbmColor)); // + if (icoInfo.hbmMask) //VC11 static analyzer warns this could be null + ::DeleteObject(icoInfo.hbmMask); //nice potential for a GDI leak! - BITMAP bmpInfo = {}; - if (::GetObject(icoInfo.hbmColor, //__in HGDIOBJ hgdiobj, - sizeof(BITMAP), //__in int cbBuffer, - &bmpInfo) != 0) // __out LPVOID lpvObject + if (icoInfo.hbmColor) //optional (for black and white bitmap) { - const int maxExtent = std::max(bmpInfo.bmWidth, bmpInfo.bmHeight); - if (0 < expectedSize && expectedSize < maxExtent) + ZEN_ON_SCOPE_EXIT(::DeleteObject(icoInfo.hbmColor)); // + + BITMAP bmpInfo = {}; + if (::GetObject(icoInfo.hbmColor, //__in HGDIOBJ hgdiobj, + sizeof(BITMAP), //__in int cbBuffer, + &bmpInfo) != 0) // __out LPVOID lpvObject { - bmpInfo.bmWidth = bmpInfo.bmWidth * expectedSize / maxExtent; //scale those Vista jumbo 256x256 icons down! - bmpInfo.bmHeight = bmpInfo.bmHeight * expectedSize / maxExtent; // + const int maxExtent = std::max(bmpInfo.bmWidth, bmpInfo.bmHeight); + if (0 < expectedSize && expectedSize < maxExtent) + { + bmpInfo.bmWidth = bmpInfo.bmWidth * expectedSize / maxExtent; //scale those Vista jumbo 256x256 icons down! + bmpInfo.bmHeight = bmpInfo.bmHeight * expectedSize / maxExtent; // + } + newIcon.SetSize(bmpInfo.bmWidth, bmpInfo.bmHeight); //wxIcon is stretched to this size } - newIcon.SetSize(bmpInfo.bmWidth, bmpInfo.bmHeight); //wxIcon is stretched to this size } } } //no stretching for now - //newIcon.SetSize(defaultSize, defaultSize); //icon is stretched to this size if referenced HICON differs + //newIcon.SetSize(expectedSize, expectedSize); //icon is stretched to this size if referenced HICON differs #elif defined FFS_LINUX // newIcon.SetPixbuf(clone.handle_); // transfer ownership!! #endif // - clone.handle_ = nullptr; // + clone.handle_ = nullptr; // return newIcon; } @@ -486,7 +490,7 @@ void WorkerThread::operator()() //thread entry //2. Initialize system image list typedef BOOL (WINAPI* FileIconInitFun)(BOOL fRestoreCache); - const SysDllFun<FileIconInitFun> fileIconInit(L"Shell32.dll", reinterpret_cast<LPCSTR>(660)); + const SysDllFun<FileIconInitFun> fileIconInit(L"Shell32.dll", reinterpret_cast<LPCSTR>(660)); //MS requires and documents this magic number assert(fileIconInit); if (fileIconInit) fileIconInit(false); //TRUE to restore the system image cache from disk; FALSE otherwise. diff --git a/lib/localization.cpp b/lib/localization.cpp index 5f9a4750..5bbb31d1 100644 --- a/lib/localization.cpp +++ b/lib/localization.cpp @@ -9,7 +9,7 @@ #include <map> #include <list> #include <iterator> -#include <wx/ffile.h> +//#include <wx/ffile.h> #include <wx/intl.h> #include <wx/msgdlg.h> #include "parse_plural.h" @@ -18,7 +18,6 @@ #include <zen/string_tools.h> #include <zen/file_traverser.h> #include "ffs_paths.h" -#include <wx+/string_conv.h> #include <zenxml/io.h> #include <zen/i18n.h> diff --git a/lib/parallel_scan.cpp b/lib/parallel_scan.cpp index 435067b2..f49fdc3e 100644 --- a/lib/parallel_scan.cpp +++ b/lib/parallel_scan.cpp @@ -10,7 +10,6 @@ #include "lock_holder.h" #include <zen/file_traverser.h> #include <zen/file_error.h> -#include <wx+/string_conv.h> #include <zen/thread.h> //includes <boost/thread.hpp> #include <zen/scope_guard.h> #include <zen/fixed_list.h> @@ -176,7 +175,7 @@ class AsyncCallback //actor pattern { public: AsyncCallback() : - notifyingThreadID(-1), + notifyingThreadID(0), textScanning(_("Scanning:")), itemsScanned(0), activeWorker(0) {} @@ -216,9 +215,9 @@ public: } } - void setNotifyingThread(int threadID) { notifyingThreadID = threadID; } //context of main thread + void setNotifyingThread(size_t threadID) { notifyingThreadID = threadID; } //context of main thread - void reportCurrentFile(const Zstring& filename, int threadID) //context of worker thread + void reportCurrentFile(const Zstring& filename, size_t threadID) //context of worker thread { if (threadID != notifyingThreadID) return; //only one thread at a time may report status @@ -227,7 +226,7 @@ public: currentStatus.clear(); } - void reportCurrentStatus(const std::wstring& status, int threadID) //context of worker thread + void reportCurrentStatus(const std::wstring& status, size_t threadID) //context of worker thread { if (threadID != notifyingThreadID) return; //only one thread may report status @@ -278,7 +277,7 @@ private: std::unique_ptr<FillBufferCallback::HandleError> errorResponse; //---- status updates ---- - volatile int notifyingThreadID; //theoretically racy, but there is nothing that could go wrong... + volatile size_t notifyingThreadID; //theoretically racy, but there is nothing that could go wrong... //CAVEAT: do NOT use boost::thread::id as long as this showstopper exists: https://svn.boost.org/trac/boost/ticket/5754 boost::mutex lockCurrentStatus; //use a different lock for current file: continue traversing while some thread may process an error Zstring currentFile; //only one of these two is filled at a time! @@ -296,7 +295,7 @@ private: struct TraverserShared { public: - TraverserShared(int threadID, + TraverserShared(size_t threadID, SymLinkHandling handleSymlinks, const HardFilter::FilterRef& filter, std::set<Zstring>& failedReads, @@ -313,7 +312,7 @@ public: std::set<Zstring>& failedReads_; //relative postfixed names of directories that could not be read (empty for root) AsyncCallback& acb_; - int threadID_; + size_t threadID_; }; @@ -459,7 +458,7 @@ DirCallback::HandleError DirCallback::onError(const std::wstring& errorText) class DstHackCallbackImpl : public DstHackCallback { public: - DstHackCallbackImpl(AsyncCallback& acb, int threadID) : + DstHackCallbackImpl(AsyncCallback& acb, size_t threadID) : acb_(acb), threadID_(threadID), textApplyingDstHack(replaceCpy(_("Encoding extended time information: %x"), L"%x", L"\n%x")) {} @@ -471,7 +470,7 @@ private: } AsyncCallback& acb_; - int threadID_; + size_t threadID_; const std::wstring textApplyingDstHack; }; #endif @@ -481,7 +480,7 @@ private: class WorkerThread { public: - WorkerThread(int threadID, + WorkerThread(size_t threadID, const std::shared_ptr<AsyncCallback>& acb, const std::vector<std::pair<DirectoryKey, DirectoryValue*>>& workload) : threadID_(threadID), @@ -523,7 +522,7 @@ public: } private: - int threadID_; + size_t threadID_; std::shared_ptr<AsyncCallback> acb_; std::vector<std::pair<DirectoryKey, DirectoryValue*>> workload_; }; @@ -563,12 +562,12 @@ void zen::fillBuffer(const std::set<DirectoryKey>& keysToRead, //in workload.push_back(std::make_pair(key, &rv.first->second)); }); - const int threadId = iter - buckets.begin(); + const size_t threadId = iter - buckets.begin(); worker.emplace_back(WorkerThread(threadId, acb, workload)); } //wait until done - int threadId = 0; + size_t threadId = 0; for (auto iter = worker.begin(); iter != worker.end(); ++iter, ++threadId) { boost::thread& wt = *iter; diff --git a/lib/process_xml.cpp b/lib/process_xml.cpp index 677618b4..39b1520b 100644 --- a/lib/process_xml.cpp +++ b/lib/process_xml.cpp @@ -5,9 +5,9 @@ // ************************************************************************** #include "process_xml.h" +#include <utility> #include <zenxml/xml.h> #include "ffs_paths.h" -#include <wx+/string_conv.h> #include <zen/file_handling.h> #include <zen/file_io.h> #include "xml_base.h" @@ -15,6 +15,8 @@ using namespace zen; using namespace xmlAccess; //functionally needed for correct overload resolution!!! +using namespace std::rel_ops; + XmlType getXmlType(const zen::XmlDoc& doc) //throw() { @@ -181,7 +183,7 @@ xmlAccess::MergeType xmlAccess::getMergeType(const std::vector<Zstring>& filenam namespace { template <class XmlCfg> -XmlCfg loadCfgImpl(const Zstring& filename, std::unique_ptr<xmlAccess::FfsXmlError>& exeption) //throw xmlAccess::FfsXmlError +XmlCfg loadCfgImpl(const Zstring& filename, std::unique_ptr<xmlAccess::FfsXmlError>& warning) //throw xmlAccess::FfsXmlError { XmlCfg cfg; try @@ -193,7 +195,7 @@ XmlCfg loadCfgImpl(const Zstring& filename, std::unique_ptr<xmlAccess::FfsXmlErr if (e.getSeverity() == xmlAccess::FfsXmlError::FATAL) throw; else - exeption.reset(new xmlAccess::FfsXmlError(e)); + warning.reset(new xmlAccess::FfsXmlError(e)); } return cfg; } @@ -207,8 +209,7 @@ void mergeConfigFilesImpl(const std::vector<Zstring>& filenames, XmlCfg& config) return; std::vector<zen::MainConfiguration> mainCfgs; - std::unique_ptr<FfsXmlError> savedException; - Zstring invalidFile; + std::unique_ptr<FfsXmlError> savedWarning; std::for_each(filenames.begin(), filenames.end(), [&](const Zstring& filename) @@ -216,23 +217,22 @@ void mergeConfigFilesImpl(const std::vector<Zstring>& filenames, XmlCfg& config) switch (getXmlType(filename)) { case XML_TYPE_GUI: - mainCfgs.push_back(loadCfgImpl<XmlGuiConfig>(filename, savedException).mainCfg); //throw xmlAccess::FfsXmlError + mainCfgs.push_back(loadCfgImpl<XmlGuiConfig>(filename, savedWarning).mainCfg); //throw xmlAccess::FfsXmlError break; case XML_TYPE_BATCH: - mainCfgs.push_back(loadCfgImpl<XmlBatchConfig>(filename, savedException).mainCfg); //throw xmlAccess::FfsXmlError + mainCfgs.push_back(loadCfgImpl<XmlBatchConfig>(filename, savedWarning).mainCfg); //throw xmlAccess::FfsXmlError break; case XML_TYPE_GLOBAL: case XML_TYPE_OTHER: - invalidFile = filename; - break; + if (!fileExists(filename)) + throw FfsXmlError(replaceCpy(_("Cannot find file %x."), L"%x", fmtFileName(filename))); + else + throw FfsXmlError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtFileName(filename))); } }); - if (mainCfgs.empty()) - throw FfsXmlError(replaceCpy(_("File %x does not contain a valid configuration."), L"%x", fmtFileName(invalidFile))); - try //...to init all non-"mainCfg" settings with first config file { xmlAccess::readConfig(filenames[0], config); //throw xmlAccess::FfsXmlError @@ -241,8 +241,8 @@ void mergeConfigFilesImpl(const std::vector<Zstring>& filenames, XmlCfg& config) config.mainCfg = merge(mainCfgs); - if (savedException.get()) //"re-throw" exception - throw* savedException; + if (savedWarning.get()) //"re-throw" exception + throw* savedWarning; } } @@ -761,7 +761,8 @@ void readConfig(const XmlIn& in, FolderPairEnh& enhPair) //########################################################### //alternate filter configuration - readConfig(in["LocalFilter"], enhPair.localFilter); + if (XmlIn inLocFilter = in["LocalFilter"]) + readConfig(inLocFilter, enhPair.localFilter); } @@ -1019,7 +1020,6 @@ void writeConfigFolderPair(const FolderPairEnh& enhPair, XmlOut& out) if (enhPair.altCmpConfig.get()) { XmlOut outAlt = outPair["CompareConfig"]; - writeConfig(*enhPair.altCmpConfig, outAlt); } //########################################################### @@ -1027,14 +1027,16 @@ void writeConfigFolderPair(const FolderPairEnh& enhPair, XmlOut& out) if (enhPair.altSyncConfig.get()) { XmlOut outAltSync = outPair["SyncConfig"]; - writeConfig(*enhPair.altSyncConfig, outAltSync); } //########################################################### //alternate filter configuration - XmlOut outFilter = outPair["LocalFilter"]; - writeConfig(enhPair.localFilter, outFilter); + if (enhPair.localFilter != FilterConfig()) //don't spam .ffs_gui file with default filter entries + { + XmlOut outFilter = outPair["LocalFilter"]; + writeConfig(enhPair.localFilter, outFilter); + } } diff --git a/lib/resolve_path.cpp b/lib/resolve_path.cpp index bb0c1f3b..57f4ff30 100644 --- a/lib/resolve_path.cpp +++ b/lib/resolve_path.cpp @@ -27,7 +27,7 @@ using namespace zen; namespace { #ifdef FFS_WIN -Zstring resolveRelativePath(Zstring relativeName) //note: ::GetFullPathName() is documented not threadsafe! +Zstring resolveRelativePath(const Zstring& relativeName) //note: ::GetFullPathName() is documented not threadsafe! { const DWORD bufferSize = 10000; std::vector<wchar_t> buffer(bufferSize); @@ -36,10 +36,10 @@ Zstring resolveRelativePath(Zstring relativeName) //note: ::GetFullPathName() is bufferSize, //__in DWORD nBufferLength, &buffer[0], //__out LPTSTR lpBuffer, nullptr); //__out LPTSTR *lpFilePart - if (charsWritten == 0 || charsWritten >= bufferSize) //theoretically, charsWritten can never be == bufferSize + if (charsWritten == 0 || charsWritten >= bufferSize) //theoretically, charsWritten cannot be == "bufferSize" return relativeName; //ERROR! Don't do anything - return Zstring(&buffer[0], charsWritten); + return removeLongPathPrefix(Zstring(&buffer[0], charsWritten)); //GetFullPathName() preserves long path prefix -> a low-level detail we don't want to leak out! } #elif defined FFS_LINUX @@ -347,7 +347,6 @@ Zstring volumePathToName(const Zstring& volumePath) //return empty string on err rv != DRIVE_CDROM) { std::vector<wchar_t> buffer(MAX_PATH + 1); - if (::GetVolumeInformation(volumePath.c_str(), //__in_opt LPCTSTR lpRootPathName, &buffer[0], //__out LPTSTR lpVolumeNameBuffer, static_cast<DWORD>(buffer.size()), //__in DWORD nVolumeNameSize, diff --git a/lib/resources.cpp b/lib/resources.cpp index 670f2cfd..0d86279b 100644 --- a/lib/resources.cpp +++ b/lib/resources.cpp @@ -10,7 +10,7 @@ #include <wx/zipstrm.h> #include <wx/image.h> #include <wx/mstream.h> -#include <wx+/string_conv.h> +//#include <wx+/string_conv.h> #include "ffs_paths.h" using namespace zen; diff --git a/lib/xml_base.cpp b/lib/xml_base.cpp index 8f8ee74d..db33059c 100644 --- a/lib/xml_base.cpp +++ b/lib/xml_base.cpp @@ -24,7 +24,7 @@ void xmlAccess::loadXmlDocument(const Zstring& filename, XmlDoc& doc) //throw Ff const std::string xmlBegin = "<?xml version="; std::vector<char> buffer(xmlBegin.size() + sizeof(zen::BYTE_ORDER_MARK_UTF8)); - FileInput inputFile(filename); //throw FileError; + FileInput inputFile(filename); //throw FileError const size_t bytesRead = inputFile.read(&buffer[0], buffer.size()); //throw FileError const std::string fileBegin(&buffer[0], bytesRead); |