From 339ed7f63798fb5ccab05fa7fb9d0d95743c9c89 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Wed, 16 Mar 2016 21:34:59 +0100 Subject: 8.0 --- zen/serialize.h | 223 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 148 insertions(+), 75 deletions(-) (limited to 'zen/serialize.h') diff --git a/zen/serialize.h b/zen/serialize.h index 77cf657e..d7b4cd22 100644 --- a/zen/serialize.h +++ b/zen/serialize.h @@ -10,7 +10,7 @@ #include #include #include "string_base.h" -#include "file_io.h" +//keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h! namespace zen @@ -32,8 +32,6 @@ class ByteArray; //ref-counted byte stream + guaranteed per class ByteArray //essentially a std::vector with ref-counted semantics, but no COW! => *almost* value type semantics, but not quite { public: - ByteArray() : buffer(std::make_shared>()) {} - typedef std::vector::value_type value_type; typedef std::vector::iterator iterator; typedef std::vector::const_iterator const_iterator; @@ -46,45 +44,78 @@ public: void resize(size_t len) { buffer->resize(len); } size_t size() const { return buffer->size(); } - bool empty() const { return buffer->empty(); } + bool empty() const { return buffer->empty(); } inline friend bool operator==(const ByteArray& lhs, const ByteArray& rhs) { return *lhs.buffer == *rhs.buffer; } private: - std::shared_ptr> buffer; //always bound! + std::shared_ptr> buffer { std::make_shared>() }; //always bound! //perf: shared_ptr indirection irrelevant: less than 1% slower! }; -//---------------------------------------------------------------------- -//functions based on binary container abstraction -template void saveBinStream(const Zstring& filepath, const BinContainer& cont, const std::function& onUpdateStatus); //throw FileError -template BinContainer loadBinStream(const Zstring& filepath, const std::function& onUpdateStatus); //throw FileError +/* +--------------------------------- +|Unbuffered Input Stream Concept| +--------------------------------- +struct UnbufferedInputStream +{ + size_t getBlockSize(); + size_t tryRead(void* buffer, size_t bytesToRead); //may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 +}; + +---------------------------------- +|Unbuffered Output Stream Concept| +---------------------------------- +struct UnbufferedOutputStream +{ + size_t getBlockSize(); + size_t tryWrite(const void* buffer, size_t bytesToWrite); //may return short! CONTRACT: bytesToWrite > 0 +}; +*/ +//functions based on unbuffered stream abstraction + +template +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, UnbufferedOutputStream& streamOut, const std::function& notifyProgress); //throw X + +template +void unbufferedSave(const BinContainer& buffer, UnbufferedOutputStream& streamOut, const std::function& notifyProgress); //throw X +template +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, const std::function& notifyProgress); //throw X /* ------------------------------ -|Binary Input Stream Concept| ------------------------------ -struct BinInputStream +------------------------------- +|Buffered Input Stream Concept| +------------------------------- +struct BufferedInputStream { - size_t read(void* data, size_t len); //return "len" bytes unless end of stream! + size_t read(void* buffer, size_t bytesToRead); //return "len" bytes unless end of stream! throw ? }; ------------------------------- -|Binary Output Stream Concept| ------------------------------- -struct BinOutputStream +-------------------------------- +|Buffered Output Stream Concept| +-------------------------------- +struct BufferedOutputStream { - void write(const void* data, size_t len); + void write(const void* buffer, size_t bytesToWrite); //throw ? }; */ +//functions based on buffered stream abstraction +template void writeNumber (BufferedOutputStream& stream, const N& num); // +template void writeContainer(BufferedOutputStream& stream, const C& str); //throw () +template < class BufferedOutputStream> void writeArray (BufferedOutputStream& stream, const void* data, size_t len); // -//binary input/output stream reference implementations: +//---------------------------------------------------------------------- +class UnexpectedEndOfStreamError {}; +template N readNumber (BufferedInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) +template C readContainer(BufferedInputStream& stream); // +template < class BufferedInputStream> void readArray (BufferedInputStream& stream, void* data, size_t len); // +//buffered input/output stream reference implementations: template struct MemoryStreamIn { - MemoryStreamIn(const BinContainer& cont) : buffer(cont), pos(0) {} //this better be cheap! + MemoryStreamIn(const BinContainer& cont) : buffer(cont) {} //this better be cheap! size_t read(void* data, size_t len) //return "len" bytes unless end of stream! { @@ -98,7 +129,7 @@ struct MemoryStreamIn private: const BinContainer buffer; - size_t pos; + size_t pos = 0; }; template @@ -118,21 +149,6 @@ private: BinContainer buffer; }; -//---------------------------------------------------------------------- -//functions based on binary stream abstraction -template -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, const std::function& onNotifyCopyStatus); //optional - -template void writeNumber (BinOutputStream& stream, const N& num); // -template void writeContainer(BinOutputStream& stream, const C& str); //throw () -template < class BinOutputStream> void writeArray (BinOutputStream& stream, const void* data, size_t len); // - -//---------------------------------------------------------------------- -class UnexpectedEndOfStreamError {}; -template N readNumber (BinInputStream& stream); //throw UnexpectedEndOfStreamError (corrupted data) -template C readContainer(BinInputStream& stream); // -template < class BinInputStream> void readArray (BinInputStream& stream, void* data, size_t len); // - @@ -141,67 +157,124 @@ template < class BinInputStream> void readArray (BinInputStream& stre //-----------------------implementation------------------------------- -template inline -void copyStream(BinInputStream& streamIn, BinOutputStream& streamOut, size_t blockSize, - const std::function& onNotifyCopyStatus) //optional +template inline +void unbufferedStreamCopy(UnbufferedInputStream& streamIn, //throw X + UnbufferedOutputStream& streamOut, // + const std::function& notifyProgress) //optional { - assert(blockSize > 0); - std::vector buffer(blockSize); + size_t unevenBytes = 0; + auto reportBytesProcessed = [&](size_t bytesReadOrWritten) + { + if (notifyProgress) + { + const size_t bytesToReport = (unevenBytes + bytesReadOrWritten) / 2; + notifyProgress(bytesToReport); //throw X! + unevenBytes = (unevenBytes + bytesReadOrWritten) - bytesToReport * 2; //unsigned arithmetics! + } + }; + + const size_t blockSizeIn = streamIn .getBlockSize(); + const size_t blockSizeOut = streamOut.getBlockSize(); + if (blockSizeIn == 0 || blockSizeOut == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + std::vector buffer; for (;;) { - const size_t bytesRead = streamIn.read(&buffer[0], buffer.size()); - streamOut.write(&buffer[0], bytesRead); + buffer.resize(buffer.size() + blockSizeIn); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSizeIn), blockSizeIn); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSizeIn + bytesRead); //caveat: unsigned arithmetics + + reportBytesProcessed(bytesRead); //throw X! - if (onNotifyCopyStatus) - onNotifyCopyStatus(bytesRead); //throw X! + size_t bytesRemaining = buffer.size(); + while (bytesRemaining >= blockSizeOut) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), blockSizeOut); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + buffer.erase(buffer.begin(), buffer.end() - bytesRemaining); - if (bytesRead != buffer.size()) //end of file + if (bytesRead == 0) //end of file break; } + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesRemaining); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + reportBytesProcessed(bytesWritten); //throw X! + } + + if (unevenBytes != 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); } -template inline -void saveBinStream(const Zstring& filepath, //throw FileError - const BinContainer& cont, - const std::function& onUpdateStatus) //optional +template inline +void unbufferedSave(const BinContainer& buffer, + UnbufferedOutputStream& streamOut, //throw X + const std::function& notifyProgress) //optional { - MemoryStreamIn streamIn(cont); - FileOutput streamOut(filepath, zen::FileOutput::ACC_OVERWRITE); //throw FileError, (ErrorTargetExisting) - if (onUpdateStatus) onUpdateStatus(0); //throw X! - copyStream(streamIn, streamOut, streamOut.optimalBlockSize(), onUpdateStatus); //throw FileError + const size_t blockSize = streamOut.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + for (size_t bytesRemaining = buffer.size(); bytesRemaining > 0;) + { + const size_t bytesToWrite = std::min(bytesRemaining, blockSize); + const size_t bytesWritten = streamOut.tryWrite(&*(buffer.end() - bytesRemaining), bytesToWrite); //throw X; may return short! CONTRACT: bytesToWrite > 0 + bytesRemaining -= bytesWritten; + if (notifyProgress) notifyProgress(bytesWritten); //throw X! + } } -template inline -BinContainer loadBinStream(const Zstring& filepath, //throw FileError - const std::function& onUpdateStatus) //optional +template inline +BinContainer unbufferedLoad(UnbufferedInputStream& streamIn, //throw X + const std::function& notifyProgress) //optional { - FileInput streamIn(filepath); //throw FileError, ErrorFileLocked - if (onUpdateStatus) onUpdateStatus(0); //throw X! - MemoryStreamOut streamOut; - copyStream(streamIn, streamOut, streamIn.optimalBlockSize(), onUpdateStatus); //throw FileError - return streamOut.ref(); + const size_t blockSize = streamIn.getBlockSize(); + if (blockSize == 0) + throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo(__LINE__)); + + static_assert(sizeof(typename BinContainer::value_type) == 1, ""); //expect: bytes + + BinContainer buffer; + for (;;) + { + buffer.resize(buffer.size() + blockSize); + const size_t bytesRead = streamIn.tryRead(&*(buffer.end() - blockSize), blockSize); //throw X; may return short, only 0 means EOF! => CONTRACT: bytesToRead > 0 + buffer.resize(buffer.size() - blockSize + bytesRead); //caveat: unsigned arithmetics + + if (notifyProgress) notifyProgress(bytesRead); //throw X! + + if (bytesRead == 0) //end of file + return buffer; + } } -template inline -void writeArray(BinOutputStream& stream, const void* data, size_t len) +template inline +void writeArray(BufferedOutputStream& stream, const void* data, size_t len) { stream.write(data, len); } -template inline -void writeNumber(BinOutputStream& stream, const N& num) +template inline +void writeNumber(BufferedOutputStream& stream, const N& num) { static_assert(IsArithmetic::value || IsSameType::value, ""); writeArray(stream, &num, sizeof(N)); } -template inline -void writeContainer(BinOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! +template inline +void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data! { const auto len = cont.size(); writeNumber(stream, static_cast(len)); @@ -210,8 +283,8 @@ void writeContainer(BinOutputStream& stream, const C& cont) //don't even conside } -template inline -void readArray(BinInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError +template inline +void readArray(BufferedInputStream& stream, void* data, size_t len) //throw UnexpectedEndOfStreamError { const size_t bytesRead = stream.read(data, len); if (bytesRead < len) @@ -219,8 +292,8 @@ void readArray(BinInputStream& stream, void* data, size_t len) //throw Unexpecte } -template inline -N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template inline +N readNumber(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { static_assert(IsArithmetic::value || IsSameType::value, ""); N num = 0; @@ -229,8 +302,8 @@ N readNumber(BinInputStream& stream) //throw UnexpectedEndOfStreamError } -template inline -C readContainer(BinInputStream& stream) //throw UnexpectedEndOfStreamError +template inline +C readContainer(BufferedInputStream& stream) //throw UnexpectedEndOfStreamError { C cont; auto strLength = readNumber(stream); -- cgit