diff options
Diffstat (limited to 'zen/serialize.h')
| -rw-r--r-- | zen/serialize.h | 223 |
1 files changed, 148 insertions, 75 deletions
diff --git a/zen/serialize.h b/zen/serialize.h index 77cf657e..d7b4cd22 100644 --- a/zen/serialize.h +++ b/zen/serialize.h @@ -10,7 +10,7 @@ #include <functional> #include <cstdint> #include "string_base.h" -#include "file_io.h" +//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h! namespace zen @@ -32,8 +32,6 @@ class ByteArray; //ref-counted byte stream + guaranteed per class ByteArray //essentially a std::vector<char> with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite { public: - ByteArray() : buffer(std::make_shared<std::vector<char>>()) {} - typedef std::vector<char>::value_type value_type; typedef std::vector<char>::iterator iterator; typedef std::vector<char>::const_iterator const_iterator; @@ -46,45 +44,78 @@ public: void resize(size_t len) { buffer->resize(len); } size_t size() const { return buffer->size(); } - bool empty() const { return buffer->empty(); } + bool empty() const { return buffer->empty(); } inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; } private: - std::shared_ptr<std::vector<char>> buffer; //always bound! + std::shared_ptr<std::vector<char>> buffer { std::make_shared<std::vector<char>>() }; //always bound! //perf: shared_ptr indirection irrelevant: less than 1% slower! }; -//---------------------------------------------------------------------- -//functions based on binary container abstraction -template <class BinContainer> void saveBinStream(const Zstring& filepath, const BinContainer& cont, const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus); //throw FileError -template <class BinContainer> BinContainer loadBinStream(const Zstring& filepath, const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus); //throw FileError +/* +--------------------------------- +|Unbuffered Input Stream Concept| +--------------------------------- +struct UnbufferedInputStream +{ + size_t getBlockSize(); + size_t tryRead(void* buffer, size_t bytesToRead); //may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 +}; + +---------------------------------- +|Unbuffered Output Stream Concept| +---------------------------------- +struct UnbufferedOutputStream +{ + size_t getBlockSize(); + size_t tryWrite(const void* buffer, size_t bytesToWrite); //may return short! CONTRACT: bytesToWrite > 0 +}; +*/ +//functions based on unbuffered stream abstraction + +template <class UnbufferedInputStream, class UnbufferedOutputStream> +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X + +template <class BinContainer, class UnbufferedOutputStream> +void unbufferedSave(const BinContainer& buffer, UnbufferedOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X +template <class BinContainer, class UnbufferedInputStream> +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, const std::function<void(std::int64_t bytesDelta)>& notifyProgress); //throw X /* ------------------------------ -|Binary Input Stream Concept| ------------------------------ -struct BinInputStream +------------------------------- +|Buffered Input Stream Concept| +------------------------------- +struct BufferedInputStream { - size_t read(void* data, size_t len); //return "len" bytes unless end of stream! + size_t read(void* buffer, size_t bytesToRead); //return "len" bytes unless end of stream! throw ? }; ------------------------------- -|Binary Output Stream Concept| ------------------------------- -struct BinOutputStream +-------------------------------- +|Buffered Output Stream Concept| +-------------------------------- +struct BufferedOutputStream { - void write(const void* data, size_t len); + void write(const void* buffer, size_t bytesToWrite); //throw ? }; */ +//functions based on buffered stream abstraction +template <class N, class BufferedOutputStream> void writeNumber (BufferedOutputStream& stream, const N& num); // +template <class C, class BufferedOutputStream> void writeContainer(BufferedOutputStream& stream, const C& str); //throw () +template < class BufferedOutputStream> void writeArray (BufferedOutputStream& stream, const void* data, size_t len); // -//binary input/output stream reference implementations: +//---------------------------------------------------------------------- +class UnexpectedEndOfStreamError {}; +template <class N, class BufferedInputStream> N readNumber (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) +template <class C, class BufferedInputStream> C readContainer(BufferedInputStream& stream); // +template < class BufferedInputStream> void readArray (BufferedInputStream& stream, void* data, size_t len); // +//buffered input/output stream reference implementations: template <class BinContainer> struct MemoryStreamIn { - MemoryStreamIn(const BinContainer& cont) : buffer(cont), pos(0) {} //this better be cheap! + MemoryStreamIn(const BinContainer& cont) : buffer(cont) {} //this better be cheap! size_t read(void* data, size_t len) //return "len" bytes unless end of stream! { @@ -98,7 +129,7 @@ struct MemoryStreamIn private: const BinContainer buffer; - size_t pos; + size_t pos = 0; }; template <class BinContainer> @@ -118,21 +149,6 @@ private: BinContainer buffer; }; -//---------------------------------------------------------------------- -//functions based on binary stream abstraction -template <class BinInputStream, class BinOutputStream> -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, const std::function<void(std::int64_t bytesDelta)>& onNotifyCopyStatus); //optional - -template <class N, class BinOutputStream> void writeNumber (BinOutputStream& stream, const N& num); // -template <class C, class BinOutputStream> void writeContainer(BinOutputStream& stream, const C& str); //throw () -template < class BinOutputStream> void writeArray (BinOutputStream& stream, const void* data, size_t len); // - -//---------------------------------------------------------------------- -class UnexpectedEndOfStreamError {}; -template <class N, class BinInputStream> N readNumber (BinInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) -template <class C, class BinInputStream> C readContainer(BinInputStream& stream); // -template < class BinInputStream> void readArray (BinInputStream& stream, void* data, size_t len); // - @@ -141,67 +157,124 @@ template < class BinInputStream> void readArray (BinInputStream& stre //-----------------------implementation------------------------------- -template <class BinInputStream, class BinOutputStream> inline -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, size_t blockSize, - const std::function<void(std::int64_t bytesDelta)>& onNotifyCopyStatus) //optional +template <class UnbufferedInputStream, class UnbufferedOutputStream> inline +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, //throw X + UnbufferedOutputStream& streamOut, // + const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional { - assert(blockSize > 0); - std::vector<char> buffer(blockSize); + size_t unevenBytes = 0; + auto reportBytesProcessed = [&](size_t bytesReadOrWritten) + { + if (notifyProgress) + { + const size_t bytesToReport = (unevenBytes + bytesReadOrWritten) / 2; + notifyProgress(bytesToReport); //throw X! + unevenBytes = (unevenBytes + bytesReadOrWritten) - bytesToReport * 2; //unsigned arithmetics! + } + }; + + const size_t blockSizeIn = streamIn .getBlockSize(); + const size_t blockSizeOut = streamOut.getBlockSize(); + if (blockSizeIn == 0 || blockSizeOut == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__)); + + std::vector<char> buffer; for (;;) { - const size_t bytesRead = streamIn.read(&buffer[0], buffer.size()); - streamOut.write(&buffer[0], bytesRead); + buffer.resize(buffer.size() + blockSizeIn); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSizeIn), blockSizeIn); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSizeIn + bytesRead); //caveat: unsigned arithmetics + + reportBytesProcessed(bytesRead); //throw X! - if (onNotifyCopyStatus) - onNotifyCopyStatus(bytesRead); //throw X! + size_t bytesRemaining = buffer.size(); + while (bytesRemaining >= blockSizeOut) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), blockSizeOut); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + buffer.erase(buffer.begin(), buffer.end() - bytesRemaining); - if (bytesRead != buffer.size()) //end of file + if (bytesRead == 0) //end of file break; } + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesRemaining); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + + if (unevenBytes != 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__)); } -template <class BinContainer> inline -void saveBinStream(const Zstring& filepath, //throw FileError - const BinContainer& cont, - const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus) //optional +template <class BinContainer, class UnbufferedOutputStream> inline +void unbufferedSave(const BinContainer& buffer, + UnbufferedOutputStream& streamOut, //throw X + const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional { - MemoryStreamIn<BinContainer> streamIn(cont); - FileOutput streamOut(filepath, zen::FileOutput::ACC_OVERWRITE); //throw FileError, (ErrorTargetExisting) - if (onUpdateStatus) onUpdateStatus(0); //throw X! - copyStream(streamIn, streamOut, streamOut.optimalBlockSize(), onUpdateStatus); //throw FileError + const size_t blockSize = streamOut.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesToWrite = std::min(bytesRemaining, blockSize); + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + if (notifyProgress) notifyProgress(bytesWritten); //throw X! + } } -template <class BinContainer> inline -BinContainer loadBinStream(const Zstring& filepath, //throw FileError - const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus) //optional +template <class BinContainer, class UnbufferedInputStream> inline +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, //throw X + const std::function<void(std::int64_t bytesDelta)>& notifyProgress) //optional { - FileInput streamIn(filepath); //throw FileError, ErrorFileLocked - if (onUpdateStatus) onUpdateStatus(0); //throw X! - MemoryStreamOut<BinContainer> streamOut; - copyStream(streamIn, streamOut, streamIn.optimalBlockSize(), onUpdateStatus); //throw FileError - return streamOut.ref(); + const size_t blockSize = streamIn.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + BinContainer buffer; + for (;;) + { + buffer.resize(buffer.size() + blockSize); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics + + if (notifyProgress) notifyProgress(bytesRead); //throw X! + + if (bytesRead == 0) //end of file + return buffer; + } } -template <class BinOutputStream> inline -void writeArray(BinOutputStream& stream, const void* data, size_t len) +template <class BufferedOutputStream> inline +void writeArray(BufferedOutputStream& stream, const void* data, size_t len) { stream.write(data, len); } -template <class N, class BinOutputStream> inline -void writeNumber(BinOutputStream& stream, const N& num) +template <class N, class BufferedOutputStream> inline +void writeNumber(BufferedOutputStream& stream, const N& num) { static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, ""); writeArray(stream, &num, sizeof(N)); } -template <class C, class BinOutputStream> inline -void writeContainer(BinOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! +template <class C, class BufferedOutputStream> inline +void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! { const auto len = cont.size(); writeNumber(stream, static_cast<std::uint32_t>(len)); @@ -210,8 +283,8 @@ void writeContainer(BinOutputStream& stream, const C& cont) //don't even conside } -template <class BinInputStream> inline -void readArray(BinInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError +template <class BufferedInputStream> inline +void readArray(BufferedInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError { const size_t bytesRead = stream.read(data, len); if (bytesRead < len) @@ -219,8 +292,8 @@ void readArray(BinInputStream& stream, void* data, size_t len) //throw Unexpecte } -template <class N, class BinInputStream> inline -N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template <class N, class BufferedInputStream> inline +N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { static_assert(IsArithmetic<N>::value || IsSameType<N, bool>::value, ""); N num = 0; @@ -229,8 +302,8 @@ N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError } -template <class C, class BinInputStream> inline -C readContainer(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template <class C, class BufferedInputStream> inline +C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { C cont; auto strLength = readNumber<std::uint32_t>(stream); |