8.0

1 files changed, 148 insertions, 75 deletions

diff --git a/zen/serialize.h b/zen/serialize.h
index 77cf657e..d7b4cd22 100644
--- a/zen/serialize.h
+++ b/zen/serialize.h
@@ -10,7 +10,7 @@
 #include <functional>
 #include <cstdint>
 #include "string_base.h"
-#include "file_io.h"
+//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h!
 
 
 namespace zen
@@ -32,8 +32,6 @@ class ByteArray;                //ref-counted       byte stream + guaranteed per
 class ByteArray //essentially a std::vector<char> with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite
 {
 public:
-    ByteArray() : buffer(std::make_shared<std::vector<char>>()) {}
-
     typedef std::vector<char>::value_type value_type;
     typedef std::vector<char>::iterator iterator;
     typedef std::vector<char>::const_iterator const_iterator;
@@ -46,45 +44,78 @@ public:
 
     void resize(size_t len) { buffer->resize(len); }
     size_t size() const { return buffer->size(); }
-    bool empty() const { return buffer->empty(); }
+    bool  empty() const { return buffer->empty(); }
 
     inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; }
 
 private:
-    std::shared_ptr<std::vector<char>> buffer; //always bound!
+    std::shared_ptr<std::vector<char>> buffer { std::make_shared<std::vector<char>>() }; //always bound!
     //perf: shared_ptr indirection irrelevant: less than 1% slower!
 };
 
-//----------------------------------------------------------------------
-//functions based on binary container abstraction
-template <class BinContainer>         void saveBinStream(const Zstring& filepath, const BinContainer& cont, const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus); //throw FileError
-template <class BinContainer> BinContainer loadBinStream(const Zstring& filepath,                           const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus); //throw FileError
+/*
+---------------------------------
+|Unbuffered Input Stream Concept|
+---------------------------------
+struct UnbufferedInputStream
+{
+    size_t getBlockSize();
+    size_t tryRead(void* buffer, size_t bytesToRead); //may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
+};
+
+----------------------------------
+|Unbuffered Output Stream Concept|
+----------------------------------
+struct UnbufferedOutputStream
+{
+    size_t getBlockSize();
+    size_t tryWrite(const void* buffer, size_t bytesToWrite); //may return short! CONTRACT: bytesToWrite > 0
+};
+*/
+//functions based on unbuffered stream abstraction
+
+template <class UnbufferedInputStream, class UnbufferedOutputStream>
+void unbufferedStreamCopy(UnbufferedInputStream& streamIn, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
+
+template <class BinContainer, class UnbufferedOutputStream>
+void unbufferedSave(const BinContainer& buffer, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
 
+template <class BinContainer, class UnbufferedInputStream>
+BinContainer unbufferedLoad(UnbufferedInputStream& streamIn,                       const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X
 
 /*
------------------------------
-|Binary Input Stream Concept|
------------------------------
-struct BinInputStream
+-------------------------------
+|Buffered Input Stream Concept|
+-------------------------------
+struct BufferedInputStream
 {
-    size_t read(void* data, size_t len); //return "len" bytes unless end of stream!
+    size_t read(void* buffer, size_t bytesToRead); //return "len" bytes unless end of stream! throw ?
 };
 
-------------------------------
-|Binary Output Stream Concept|
-------------------------------
-struct BinOutputStream
+--------------------------------
+|Buffered Output Stream Concept|
+--------------------------------
+struct BufferedOutputStream
 {
-    void write(const void* data, size_t len);
+    void write(const void* buffer, size_t bytesToWrite); //throw ?
 };
 */
+//functions based on buffered stream abstraction
+template <class N, class BufferedOutputStream> void writeNumber   (BufferedOutputStream& stream, const N& num);                 //
+template <class C, class BufferedOutputStream> void writeContainer(BufferedOutputStream& stream, const C& str);                 //throw ()
+template <         class BufferedOutputStream> void writeArray    (BufferedOutputStream& stream, const void* data, size_t len); //
 
-//binary input/output stream reference implementations:
+//----------------------------------------------------------------------
+class UnexpectedEndOfStreamError {};
+template <class N, class BufferedInputStream> N    readNumber   (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data)
+template <class C, class BufferedInputStream> C    readContainer(BufferedInputStream& stream); //
+template <         class BufferedInputStream> void readArray    (BufferedInputStream& stream, void* data, size_t len); //
 
+//buffered input/output stream reference implementations:
 template <class BinContainer>
 struct MemoryStreamIn
 {
-    MemoryStreamIn(const BinContainer& cont) : buffer(cont), pos(0) {} //this better be cheap!
+    MemoryStreamIn(const BinContainer& cont) : buffer(cont) {} //this better be cheap!
 
     size_t read(void* data, size_t len) //return "len" bytes unless end of stream!
     {
@@ -98,7 +129,7 @@ struct MemoryStreamIn
 
 private:
     const BinContainer buffer;
-    size_t pos;
+    size_t pos = 0;
 };
 
 template <class BinContainer>
@@ -118,21 +149,6 @@ private:
     BinContainer buffer;
 };
 
-//----------------------------------------------------------------------
-//functions based on binary stream abstraction
-template <class BinInputStream, class BinOutputStream>
-void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& onNotifyCopyStatus); //optional
-
-template <class N, class BinOutputStream> void writeNumber   (BinOutputStream& stream, const N& num);                 //
-template <class C, class BinOutputStream> void writeContainer(BinOutputStream& stream, const C& str);                 //throw ()
-template <         class BinOutputStream> void writeArray    (BinOutputStream& stream, const void* data, size_t len); //
-
-//----------------------------------------------------------------------
-class UnexpectedEndOfStreamError {};
-template <class N, class BinInputStream> N    readNumber   (BinInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data)
-template <class C, class BinInputStream> C    readContainer(BinInputStream& stream); //
-template <         class BinInputStream> void readArray    (BinInputStream& stream, void* data, size_t len); //
-
 
 
 
@@ -141,67 +157,124 @@ template <         class BinInputStream> void readArray    (BinInputStream& stre
 
 
 //-----------------------implementation-------------------------------
-template <class BinInputStream, class BinOutputStream> inline
-void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, size_t blockSize,
-                const std::function<void(std::int64_t bytesDelta)>& onNotifyCopyStatus) //optional
+template <class UnbufferedInputStream, class UnbufferedOutputStream> inline
+void unbufferedStreamCopy(UnbufferedInputStream& streamIn,   //throw X
+                          UnbufferedOutputStream& streamOut, //
+                          const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
 {
-    assert(blockSize > 0);
-    std::vector<char> buffer(blockSize);
+    size_t unevenBytes = 0;
+    auto reportBytesProcessed = [&](size_t bytesReadOrWritten)
+    {
+        if (notifyProgress)
+        {
+            const size_t bytesToReport = (unevenBytes + bytesReadOrWritten) / 2;
+            notifyProgress(bytesToReport); //throw X!
+            unevenBytes = (unevenBytes + bytesReadOrWritten) - bytesToReport * 2; //unsigned arithmetics!
+        }
+    };
+
+    const size_t blockSizeIn  = streamIn .getBlockSize();
+    const size_t blockSizeOut = streamOut.getBlockSize();
+    if (blockSizeIn == 0 || blockSizeOut == 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
+
+    std::vector<char> buffer;
     for (;;)
     {
-        const size_t bytesRead = streamIn.read(&buffer[0], buffer.size());
-        streamOut.write(&buffer[0], bytesRead);
+        buffer.resize(buffer.size() + blockSizeIn);
+        const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSizeIn), blockSizeIn); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
+        buffer.resize(buffer.size() - blockSizeIn + bytesRead); //caveat: unsigned arithmetics
+
+        reportBytesProcessed(bytesRead); //throw X!
 
-        if (onNotifyCopyStatus)
-            onNotifyCopyStatus(bytesRead); //throw X!
+        size_t bytesRemaining = buffer.size();
+        while (bytesRemaining >= blockSizeOut)
+        {
+            const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), blockSizeOut); //throw X; may return short! CONTRACT: bytesToWrite > 0
+            bytesRemaining -= bytesWritten;
+            reportBytesProcessed(bytesWritten); //throw X!
+        }
+        buffer.erase(buffer.begin(), buffer.end() - bytesRemaining);
 
-        if (bytesRead != buffer.size()) //end of file
+        if (bytesRead == 0) //end of file
             break;
     }
+
+    for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;)
+    {
+        const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesRemaining); //throw X; may return short! CONTRACT: bytesToWrite > 0
+        bytesRemaining -= bytesWritten;
+        reportBytesProcessed(bytesWritten); //throw X!
+    }
+
+    if (unevenBytes != 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
 }
 
 
-template <class BinContainer> inline
-void saveBinStream(const Zstring& filepath, //throw FileError
-                   const BinContainer& cont,
-                   const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus) //optional
+template <class BinContainer, class UnbufferedOutputStream> inline
+void unbufferedSave(const BinContainer& buffer,
+                    UnbufferedOutputStream& streamOut, //throw X
+                    const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
 {
-    MemoryStreamIn<BinContainer> streamIn(cont);
-    FileOutput streamOut(filepath, zen::FileOutput::ACC_OVERWRITE); //throw FileError, (ErrorTargetExisting)
-    if (onUpdateStatus) onUpdateStatus(0); //throw X!
-    copyStream(streamIn, streamOut, streamOut.optimalBlockSize(), onUpdateStatus); //throw FileError
+    const size_t blockSize = streamOut.getBlockSize();
+    if (blockSize == 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
+
+    static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
+
+    for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;)
+    {
+        const size_t bytesToWrite = std::min(bytesRemaining, blockSize);
+        const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0
+        bytesRemaining -= bytesWritten;
+        if (notifyProgress) notifyProgress(bytesWritten); //throw X!
+    }
 }
 
 
-template <class BinContainer> inline
-BinContainer loadBinStream(const Zstring& filepath, //throw FileError
-                           const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus) //optional
+template <class BinContainer, class UnbufferedInputStream> inline
+BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, //throw X
+                            const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional
 {
-    FileInput streamIn(filepath); //throw FileError, ErrorFileLocked
-    if (onUpdateStatus) onUpdateStatus(0); //throw X!
-    MemoryStreamOut<BinContainer> streamOut;
-    copyStream(streamIn, streamOut, streamIn.optimalBlockSize(), onUpdateStatus); //throw FileError
-    return streamOut.ref();
+    const size_t blockSize = streamIn.getBlockSize();
+    if (blockSize == 0)
+        throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
+
+    static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes
+
+    BinContainer buffer;
+    for (;;)
+    {
+        buffer.resize(buffer.size() + blockSize);
+        const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0
+        buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics
+
+        if (notifyProgress) notifyProgress(bytesRead); //throw X!
+
+        if (bytesRead == 0) //end of file
+            return buffer;
+    }
 }
 
 
-template <class BinOutputStream> inline
-void writeArray(BinOutputStream& stream, const void* data, size_t len)
+template <class BufferedOutputStream> inline
+void writeArray(BufferedOutputStream& stream, const void* data, size_t len)
 {
     stream.write(data, len);
 }
 
 
-template <class N, class BinOutputStream> inline
-void writeNumber(BinOutputStream& stream, const N& num)
+template <class N, class BufferedOutputStream> inline
+void writeNumber(BufferedOutputStream& stream, const N& num)
 {
     static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "");
     writeArray(stream, &num, sizeof(N));
 }
 
 
-template <class C, class BinOutputStream> inline
-void writeContainer(BinOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data!
+template <class C, class BufferedOutputStream> inline
+void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data!
 {
     const auto len = cont.size();
     writeNumber(stream, static_cast<std::uint32_t>(len));
@@ -210,8 +283,8 @@ void writeContainer(BinOutputStream& stream, const C& cont) //don't even conside
 }
 
 
-template <class BinInputStream> inline
-void readArray(BinInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError
+template <class BufferedInputStream> inline
+void readArray(BufferedInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError
 {
     const size_t bytesRead = stream.read(data, len);
     if (bytesRead < len)
@@ -219,8 +292,8 @@ void readArray(BinInputStream& stream, void* data, size_t len) //throw Unexpecte
 }
 
 
-template <class N, class BinInputStream> inline
-N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError
+template <class N, class BufferedInputStream> inline
+N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
 {
     static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, "");
     N num = 0;
@@ -229,8 +302,8 @@ N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError
 }
 
 
-template <class C, class BinInputStream> inline
-C readContainer(BinInputStream& stream) //throw UnexpectedEndOfStreamError
+template <class C, class BufferedInputStream> inline
+C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError
 {
     C cont;
     auto strLength = readNumber<std::uint32_t>(stream);