1 files changed, 280 insertions, 327 deletions
diff --git a/zen/serialize.h b/zen/serialize.h
index 290d9200..bb2f7a45 100644..100755
--- a/zen/serialize.h
+++ b/zen/serialize.h
@@ -1,327 +1,280 @@
-// *****************************************************************************
-// * This file is part of the FreeFileSync project. It is distributed under    *
-// * GNU General Public License: http://www.gnu.org/licenses/gpl-3.0           *
-// * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved *
-// *****************************************************************************
-
-#ifndef SERIALIZE_H_839405783574356
-#define SERIALIZE_H_839405783574356
-
-#include <functional>
-#include <cstdint>
-#include "string_base.h"
-//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h!
-
-
-namespace zen
-{
-//high-performance unformatted serialization (avoiding wxMemoryOutputStream/wxMemoryInputStream inefficiencies)
-
-/*
---------------------------
-|Binary Container Concept|
---------------------------
-binary container for data storage: must support "basic" std::vector interface (e.g. std::vector<char>, std::string, Zbase<char>)
-*/
-
-//binary container reference implementations
-using Utf8String = Zbase<char>; //ref-counted + COW text stream + guaranteed performance: exponential growth
-class ByteArray;                //ref-counted       byte stream + guaranteed performance: exponential growth -> no COW, but 12% faster than Utf8String (due to no null-termination?)
-
-
-class ByteArray //essentially a std::vector<char> with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite
-{
-public:
-    using value_type     = std::vector<char>::value_type;
-    using iterator       = std::vector<char>::iterator;
-    using const_iterator = std::vector<char>::const_iterator;
-
-    iterator begin() { return buffer->begin(); }
-    iterator end  () { return buffer->end  (); }
-
-    const_iterator begin() const { return buffer->begin(); }
-    const_iterator end  () const { return buffer->end  (); }
-
-    void resize(size_t len) { buffer->resize(len); }
-    size_t size() const { return buffer->size(); }
-    bool  empty() const { return buffer->empty(); }
-
-    inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; }
-
-private:
-    std::shared_ptr<std::vector<char>> buffer { std::make_shared<std::vector<char>>() }; //always bound!
-    //perf: shared_ptr indirection irrelevant: less than 1% slower!
-};
-
-/*
----------------------------------
-|Unbuffered Input Stream Concept|
----------------------------------
-struct UnbufferedInputStream
-{
-    size_t getBlockSize();
-    size_t tryRead(void* buffer, size_t bytesToRead); //may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
-};
-
-----------------------------------
-|Unbuffered Output Stream Concept|
-----------------------------------
-struct UnbufferedOutputStream
-{
-    size_t getBlockSize();
-    size_t tryWrite(const void* buffer, size_t bytesToWrite); //may return short! CONTRACT: bytesToWrite > 0
-};
-*/
-
-//functions based on unbuffered stream abstraction
-template <class UnbufferedInputStream, class UnbufferedOutputStream>
-void unbufferedStreamCopy(UnbufferedInputStream& streamIn, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
-
-template <class BinContainer, class UnbufferedOutputStream>
-void unbufferedSave(const BinContainer& buffer, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
-
-template <class BinContainer, class UnbufferedInputStream>
-BinContainer unbufferedLoad(UnbufferedInputStream& streamIn,                       const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
-
-/*
--------------------------------
-|Buffered Input Stream Concept|
--------------------------------
-struct BufferedInputStream
-{
-    size_t read(void* buffer, size_t bytesToRead); //return "len" bytes unless end of stream! throw ?
-};
-
---------------------------------
-|Buffered Output Stream Concept|
---------------------------------
-struct BufferedOutputStream
-{
-    void write(const void* buffer, size_t bytesToWrite); //throw ?
-};
-*/
-//functions based on buffered stream abstraction
-template <class N, class BufferedOutputStream> void writeNumber   (BufferedOutputStream& stream, const N& num);                 //
-template <class C, class BufferedOutputStream> void writeContainer(BufferedOutputStream& stream, const C& str);                 //throw ()
-template <         class BufferedOutputStream> void writeArray    (BufferedOutputStream& stream, const void* data, size_t len); //
-//----------------------------------------------------------------------
-class UnexpectedEndOfStreamError {};
-template <class N, class BufferedInputStream> N    readNumber   (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data)
-template <class C, class BufferedInputStream> C    readContainer(BufferedInputStream& stream); //
-template <         class BufferedInputStream> void readArray    (BufferedInputStream& stream, void* data, size_t len); //
-
-//buffered input/output stream reference implementations:
-template <class BinContainer>
-struct MemoryStreamIn
-{
-    MemoryStreamIn(const BinContainer& cont) : buffer_(cont) {} //this better be cheap!
-
-    size_t read(void* data, size_t len) //return "len" bytes unless end of stream!
-    {
-        static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
-        const size_t bytesRead = std::min(len, buffer_.size() - pos_);
-        auto itFirst = buffer_.begin() + pos_;
-        std::copy(itFirst, itFirst + bytesRead, static_cast<char*>(data));
-        pos_ += bytesRead;
-        return bytesRead;
-    }
-
-    size_t pos() const { return pos_; }
-
-private:
-    const BinContainer buffer_;
-    size_t pos_ = 0;
-};
-
-template <class BinContainer>
-struct MemoryStreamOut
-{
-    void write(const void* data, size_t len)
-    {
-        static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
-        const size_t oldSize = buffer_.size();
-        buffer_.resize(oldSize + len);
-        std::copy(static_cast<const char*>(data), static_cast<const char*>(data) + len, buffer_.begin() + oldSize);
-    }
-
-    const BinContainer& ref() const { return buffer_; }
-
-private:
-    BinContainer buffer_;
-};
-
-
-
-
-
-
-
-
-//-----------------------implementation-------------------------------
-template <class UnbufferedInputStream, class UnbufferedOutputStream> inline
-void unbufferedStreamCopy(UnbufferedInputStream& streamIn,   //throw X
-                          UnbufferedOutputStream& streamOut, //
-                          const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
-{
-    size_t unevenBytes = 0;
-    auto reportBytesProcessed = [&](size_t bytesReadOrWritten)
-    {
-        if (notifyProgress)
-        {
-            const size_t bytesToReport = (unevenBytes + bytesReadOrWritten) / 2;
-            notifyProgress(bytesToReport); //throw X!
-            unevenBytes = (unevenBytes + bytesReadOrWritten) - bytesToReport * 2; //unsigned arithmetics!
-        }
-    };
-
-    const size_t blockSizeIn  = streamIn .getBlockSize();
-    const size_t blockSizeOut = streamOut.getBlockSize();
-    if (blockSizeIn == 0 || blockSizeOut == 0)
-        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
-
-    std::vector<char> buffer;
-    for (;;)
-    {
-        buffer.resize(buffer.size() + blockSizeIn);
-        const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSizeIn), blockSizeIn); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
-        buffer.resize(buffer.size() - blockSizeIn + bytesRead); //caveat: unsigned arithmetics
-
-        reportBytesProcessed(bytesRead); //throw X!
-
-        size_t bytesRemaining = buffer.size();
-        while (bytesRemaining >= blockSizeOut)
-        {
-            const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), blockSizeOut); //throw X; may return short! CONTRACT: bytesToWrite > 0
-            bytesRemaining -= bytesWritten;
-            reportBytesProcessed(bytesWritten); //throw X!
-        }
-        buffer.erase(buffer.begin(), buffer.end() - bytesRemaining);
-
-        if (bytesRead == 0) //end of file
-            break;
-    }
-
-    for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;)
-    {
-        const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesRemaining); //throw X; may return short! CONTRACT: bytesToWrite > 0
-        bytesRemaining -= bytesWritten;
-        reportBytesProcessed(bytesWritten); //throw X!
-    }
-
-    if (unevenBytes != 0)
-        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
-}
-
-
-template <class BinContainer, class UnbufferedOutputStream> inline
-void unbufferedSave(const BinContainer& buffer,
-                    UnbufferedOutputStream& streamOut, //throw X
-                    const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
-{
-    const size_t blockSize = streamOut.getBlockSize();
-    if (blockSize == 0)
-        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
-
-    static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
-
-    for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;)
-    {
-        const size_t bytesToWrite = std::min(bytesRemaining, blockSize);
-        const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0
-        bytesRemaining -= bytesWritten;
-        if (notifyProgress) notifyProgress(bytesWritten); //throw X!
-    }
-}
-
-
-template <class BinContainer, class UnbufferedInputStream> inline
-BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, //throw X
-                            const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
-{
-    const size_t blockSize = streamIn.getBlockSize();
-    if (blockSize == 0)
-        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
-
-    static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
-
-    BinContainer buffer;
-    for (;;)
-    {
-        buffer.resize(buffer.size() + blockSize);
-        const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
-        buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics
-
-        if (notifyProgress) notifyProgress(bytesRead); //throw X!
-
-        if (bytesRead == 0) //end of file
-            return buffer;
-    }
-}
-
-
-template <class BufferedOutputStream> inline
-void writeArray(BufferedOutputStream& stream, const void* data, size_t len)
-{
-    stream.write(data, len);
-}
-
-
-template <class N, class BufferedOutputStream> inline
-void writeNumber(BufferedOutputStream& stream, const N& num)
-{
-    static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "not a number!");
-    writeArray(stream, &num, sizeof(N));
-}
-
-
-template <class C, class BufferedOutputStream> inline
-void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data!
-{
-    const auto len = cont.size();
-    writeNumber(stream, static_cast<std::uint32_t>(len));
-    if (len > 0)
-        writeArray(stream, &*cont.begin(), sizeof(typename C::value_type) * len); //don't use c_str(), but access uniformly via STL interface
-}
-
-
-template <class BufferedInputStream> inline
-void readArray(BufferedInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError
-{
-    const size_t bytesRead = stream.read(data, len);
-    if (bytesRead < len)
-        throw UnexpectedEndOfStreamError();
-}
-
-
-template <class N, class BufferedInputStream> inline
-N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
-{
-    static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "");
-    N num = 0;
-    readArray(stream, &num, sizeof(N)); //throw UnexpectedEndOfStreamError
-    return num;
-}
-
-
-template <class C, class BufferedInputStream> inline
-C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
-{
-    C cont;
-    auto strLength = readNumber<std::uint32_t>(stream);
-    if (strLength > 0)
-    {
-        try
-        {
-            cont.resize(strLength); //throw std::bad_alloc
-        }
-        catch (std::bad_alloc&) //most likely this is due to data corruption!
-        {
-            throw UnexpectedEndOfStreamError();
-        }
-        readArray(stream, &*cont.begin(), sizeof(typename C::value_type) * strLength); //throw UnexpectedEndOfStreamError
-    }
-    return cont;
-}
-}
-
-#endif //SERIALIZE_H_839405783574356
+// *****************************************************************************
+// * This file is part of the FreeFileSync project. It is distributed under    *
+// * GNU General Public License: http://www.gnu.org/licenses/gpl-3.0           *
+// * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved *
+// *****************************************************************************
+
+#ifndef SERIALIZE_H_839405783574356
+#define SERIALIZE_H_839405783574356
+
+#include <functional>
+#include <cstdint>
+#include "string_base.h"
+//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h!
+
+
+namespace zen
+{
+//high-performance unformatted serialization (avoiding wxMemoryOutputStream/wxMemoryInputStream inefficiencies)
+
+/*
+--------------------------
+|Binary Container Concept|
+--------------------------
+binary container for data storage: must support "basic" std::vector interface (e.g. std::vector<char>, std::string, Zbase<char>)
+*/
+
+//binary container reference implementations
+using Utf8String = Zbase<char>; //ref-counted + COW text stream + guaranteed performance: exponential growth
+class ByteArray;                //ref-counted       byte stream + guaranteed performance: exponential growth -> no COW, but 12% faster than Utf8String (due to no null-termination?)
+
+
+class ByteArray //essentially a std::vector<char> with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite
+{
+public:
+    using value_type     = std::vector<char>::value_type;
+    using iterator       = std::vector<char>::iterator;
+    using const_iterator = std::vector<char>::const_iterator;
+
+    iterator begin() { return buffer->begin(); }
+    iterator end  () { return buffer->end  (); }
+
+    const_iterator begin() const { return buffer->begin(); }
+    const_iterator end  () const { return buffer->end  (); }
+
+    void resize(size_t len) { buffer->resize(len); }
+    size_t size() const { return buffer->size(); }
+    bool  empty() const { return buffer->empty(); }
+
+    inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; }
+
+private:
+    std::shared_ptr<std::vector<char>> buffer { std::make_shared<std::vector<char>>() }; //always bound!
+    //perf: shared_ptr indirection irrelevant: less than 1% slower!
+};
+
+/*
+-------------------------------
+|Buffered Input Stream Concept|
+-------------------------------
+struct BufferedInputStream
+{
+    size_t read(void* buffer, size_t bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream!
+
+Optional: support stream-copying
+--------------------------------
+    size_t getBlockSize() const;
+    const IOCallback& notifyUnbufferedIO
+};
+
+--------------------------------
+|Buffered Output Stream Concept|
+--------------------------------
+struct BufferedOutputStream
+{
+    void write(const void* buffer, size_t bytesToWrite); //throw X
+
+Optional: support stream-copying
+--------------------------------
+    const IOCallback& notifyUnbufferedIO
+};
+*/
+using IOCallback = std::function<void(int64_t bytesDelta)>; //throw X
+
+
+//functions based on buffered stream abstraction
+template <class BufferedInputStream, class BufferedOutputStream>
+void bufferedStreamCopy(BufferedInputStream& streamIn, BufferedOutputStream& streamOut); //throw X
+
+template <class BinContainer, class BufferedInputStream> BinContainer
+bufferedLoad(BufferedInputStream& streamIn); //throw X
+
+template <class N, class BufferedOutputStream> void writeNumber   (BufferedOutputStream& stream, const N& num);                   //
+template <class C, class BufferedOutputStream> void writeContainer(BufferedOutputStream& stream, const C& str);                   //throw ()
+template <         class BufferedOutputStream> void writeArray    (BufferedOutputStream& stream, const void* buffer, size_t len); //
+//----------------------------------------------------------------------
+class UnexpectedEndOfStreamError {};
+template <class N, class BufferedInputStream> N    readNumber   (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data)
+template <class C, class BufferedInputStream> C    readContainer(BufferedInputStream& stream); //
+template <         class BufferedInputStream> void readArray    (BufferedInputStream& stream, void* buffer, size_t len); //
+
+
+struct IOCallbackDivider
+{
+    IOCallbackDivider(const IOCallback& notifyUnbufferedIO, int64_t& totalUnbufferedIO) : totalUnbufferedIO_(totalUnbufferedIO), notifyUnbufferedIO_(notifyUnbufferedIO) {}
+
+    void operator()(int64_t bytesDelta)
+    {
+        if (notifyUnbufferedIO_) notifyUnbufferedIO_((totalUnbufferedIO_ - totalUnbufferedIO_ / 2 * 2 + bytesDelta) / 2); //throw X!
+        totalUnbufferedIO_ += bytesDelta;
+    }
+
+private:
+    int64_t& totalUnbufferedIO_;
+    const IOCallback& notifyUnbufferedIO_;
+};
+
+//buffered input/output stream reference implementations:
+template <class BinContainer>
+struct MemoryStreamIn
+{
+    MemoryStreamIn(const BinContainer& cont) : buffer_(cont) {} //this better be cheap!
+
+    size_t read(void* buffer, size_t bytesToRead) //return "bytesToRead" bytes unless end of stream!
+    {
+        static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
+        const size_t bytesRead = std::min(bytesToRead, buffer_.size() - pos_);
+        auto itFirst = buffer_.begin() + pos_;
+        std::copy(itFirst, itFirst + bytesRead, static_cast<char*>(buffer));
+        pos_ += bytesRead;
+        return bytesRead;
+    }
+
+    size_t pos() const { return pos_; }
+
+private:
+    MemoryStreamIn           (const MemoryStreamIn&) = delete;
+    MemoryStreamIn& operator=(const MemoryStreamIn&) = delete;
+
+    const BinContainer buffer_;
+    size_t pos_ = 0;
+};
+
+template <class BinContainer>
+struct MemoryStreamOut
+{
+    MemoryStreamOut() = default;
+
+    void write(const void* buffer, size_t bytesToWrite)
+    {
+        static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
+        const size_t oldSize = buffer_.size();
+        buffer_.resize(oldSize + bytesToWrite);
+        std::copy(static_cast<const char*>(buffer), static_cast<const char*>(buffer) + bytesToWrite, buffer_.begin() + oldSize);
+    }
+
+    const BinContainer& ref() const { return buffer_; }
+
+private:
+    MemoryStreamOut           (const MemoryStreamOut&) = delete;
+    MemoryStreamOut& operator=(const MemoryStreamOut&) = delete;
+
+    BinContainer buffer_;
+};
+
+
+
+
+
+
+
+
+//-----------------------implementation-------------------------------
+template <class BufferedInputStream, class BufferedOutputStream> inline
+void bufferedStreamCopy(BufferedInputStream& streamIn,   //throw X
+                        BufferedOutputStream& streamOut) //
+{
+    const size_t blockSize = streamIn.getBlockSize();
+    if (blockSize == 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
+
+    std::vector<char> buffer(blockSize);
+    for (;;)
+    {
+        const size_t bytesRead = streamIn.read(&buffer[0], blockSize); //throw X; return "bytesToRead" bytes unless end of stream!
+        streamOut.write(&buffer[0], bytesRead); //throw X
+
+        if (bytesRead < blockSize) //end of file
+            break;
+    }
+}
+
+
+template <class BinContainer, class BufferedInputStream> inline
+BinContainer bufferedLoad(BufferedInputStream& streamIn) //throw X
+{
+    static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
+
+    const size_t blockSize = streamIn.getBlockSize();
+    if (blockSize == 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
+
+    BinContainer buffer;
+    for (;;)
+    {
+        buffer.resize(buffer.size() + blockSize);
+        const size_t bytesRead = streamIn.read(&*(buffer.end() - blockSize), blockSize); //throw X; return "bytesToRead" bytes unless end of stream!
+        buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics
+
+        if (bytesRead < blockSize) //end of file
+            return buffer;
+    }
+}
+
+
+template <class BufferedOutputStream> inline
+void writeArray(BufferedOutputStream& stream, const void* buffer, size_t len)
+{
+    stream.write(buffer, len);
+}
+
+
+template <class N, class BufferedOutputStream> inline
+void writeNumber(BufferedOutputStream& stream, const N& num)
+{
+    static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "not a number!");
+    writeArray(stream, &num, sizeof(N));
+}
+
+
+template <class C, class BufferedOutputStream> inline
+void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data!
+{
+    const auto len = cont.size();
+    writeNumber(stream, static_cast<uint32_t>(len));
+    if (len > 0)
+        writeArray(stream, &*cont.begin(), sizeof(typename C::value_type) * len); //don't use c_str(), but access uniformly via STL interface
+}
+
+
+template <class BufferedInputStream> inline
+void readArray(BufferedInputStream& stream, void* buffer, size_t len) //throw UnexpectedEndOfStreamError
+{
+    const size_t bytesRead = stream.read(buffer, len);
+    if (bytesRead < len)
+        throw UnexpectedEndOfStreamError();
+}
+
+
+template <class N, class BufferedInputStream> inline
+N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
+{
+    static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "");
+    N num = 0;
+    readArray(stream, &num, sizeof(N)); //throw UnexpectedEndOfStreamError
+    return num;
+}
+
+
+template <class C, class BufferedInputStream> inline
+C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
+{
+    C cont;
+    auto strLength = readNumber<uint32_t>(stream); //throw UnexpectedEndOfStreamError
+    if (strLength > 0)
+    {
+        try
+        {
+            cont.resize(strLength); //throw std::bad_alloc
+        }
+        catch (std::bad_alloc&) //most likely this is due to data corruption!
+        {
+            throw UnexpectedEndOfStreamError();
+        }
+        readArray(stream, &*cont.begin(), sizeof(typename C::value_type) * strLength); //throw UnexpectedEndOfStreamError
+    }
+    return cont;
+}
+}
+
+#endif //SERIALIZE_H_839405783574356