From 01eb8253196672c969a39587e90b49321a182428 Mon Sep 17 00:00:00 2001 From: Daniel Wilhelm Date: Fri, 18 Apr 2014 17:19:14 +0200 Subject: 5.1 --- zen/utf8.h | 129 ++++++++++++++++++++++++++----------------------------------- 1 file changed, 55 insertions(+), 74 deletions(-) (limited to 'zen/utf8.h') diff --git a/zen/utf8.h b/zen/utf8.h index 3d97ca7a..242b729f 100644 --- a/zen/utf8.h +++ b/zen/utf8.h @@ -8,13 +8,15 @@ #ifndef STRING_UTF8_HEADER_01832479146991573473545 #define STRING_UTF8_HEADER_01832479146991573473545 +#include #include #include "string_tools.h" //copyStringTo namespace zen { //convert any(!) "string-like" object into target string by applying a UTF8 conversion (but only if necessary!) -template TargetString utf8CvrtTo(const SourceString& str); +template +TargetString utf8CvrtTo(const SourceString& str); //convert wide to utf8 string; example: std::string tmp = toUtf8(L"abc"); template @@ -62,7 +64,9 @@ const char BYTE_ORDER_MARK_UTF8[] = "\xEF\xBB\xBF"; //----------------------- implementation ---------------------------------- namespace implementation { -typedef unsigned int CodePoint; //must be at least four bytes +typedef std::uint_fast32_t CodePoint; //must be at least four bytes +typedef std::uint_fast16_t Char16; //we need an unsigned type +typedef unsigned char Char8; const CodePoint CODE_POINT_MAX = 0x10ffff; @@ -73,31 +77,28 @@ const CodePoint LOW_SURROGATE = 0xdc00; const CodePoint LOW_SURROGATE_MAX = 0xdfff; -template inline -OutputIterator codePointToUtf16(CodePoint cp, OutputIterator result) //http://en.wikipedia.org/wiki/UTF-16 +template inline +void codePointToUtf16(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char16 { - typedef unsigned short Char16; //this isn't necessarily 16 bit, but all we need is an unsigned type - + //http://en.wikipedia.org/wiki/UTF-16 assert(cp < HIGH_SURROGATE || LOW_SURROGATE_MAX < cp); //code points [0xd800, 0xdfff] are not allowed for UTF-16 assert(cp <= CODE_POINT_MAX); if (cp < 0x10000) - *result++ = static_cast(cp); + writeOutput(static_cast(cp)); else { cp -= 0x10000; - *result++ = static_cast((cp >> 10) + HIGH_SURROGATE); - *result++ = static_cast((cp & 0x3ff) + LOW_SURROGATE); + writeOutput(static_cast((cp >> 10) + HIGH_SURROGATE)); + writeOutput(static_cast((cp & 0x3ff) + LOW_SURROGATE)); } - return result; } template inline -Function utf16ToCodePoint(CharIterator first, CharIterator last, Function f) //f is a unary function taking a CodePoint as single parameter +void utf16ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint { assert_static(sizeof(typename std::iterator_traits::value_type) == 2); - typedef unsigned short Char16; //this isn't necessarily 16 bit, but all we need is an unsigned type for ( ; first != last; ++first) { @@ -107,7 +108,7 @@ Function utf16ToCodePoint(CharIterator first, CharIterator last, Function f) //f if (++first == last) { assert(false); //low surrogate expected - break; + return; } assert(LOW_SURROGATE <= static_cast(*first) && static_cast(*first) <= LOW_SURROGATE_MAX); //low surrogate expected cp = ((cp - HIGH_SURROGATE) << 10) + static_cast(*first) - LOW_SURROGATE + 0x10000; @@ -115,40 +116,37 @@ Function utf16ToCodePoint(CharIterator first, CharIterator last, Function f) //f else assert(cp < LOW_SURROGATE || LOW_SURROGATE_MAX < cp); //NO low surrogate expected - f(cp); + writeOutput(cp); } - return f; } -template inline -OutputIterator codePointToUtf8(CodePoint cp, OutputIterator result) //http://en.wikipedia.org/wiki/UTF-8 +template inline +void codePointToUtf8(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a Char8 { - typedef unsigned char Char8; - - assert(cp <= CODE_POINT_MAX); + //http://en.wikipedia.org/wiki/UTF-8 if (cp < 0x80) - *result++ = static_cast(cp); + writeOutput(static_cast(cp)); else if (cp < 0x800) { - *result++ = static_cast((cp >> 6 ) | 0xc0); - *result++ = static_cast((cp & 0x3f) | 0x80); + writeOutput(static_cast((cp >> 6 ) | 0xc0)); + writeOutput(static_cast((cp & 0x3f) | 0x80)); } else if (cp < 0x10000) { - *result++ = static_cast((cp >> 12 ) | 0xe0); - *result++ = static_cast(((cp >> 6) & 0x3f) | 0x80); - *result++ = static_cast((cp & 0x3f ) | 0x80); + writeOutput(static_cast((cp >> 12 ) | 0xe0)); + writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); + writeOutput(static_cast((cp & 0x3f ) | 0x80)); } else { - *result++ = static_cast((cp >> 18 ) | 0xf0); - *result++ = static_cast(((cp >> 12) & 0x3f) | 0x80); - *result++ = static_cast(((cp >> 6) & 0x3f) | 0x80); - *result++ = static_cast((cp & 0x3f ) | 0x80); + assert(cp <= CODE_POINT_MAX); + writeOutput(static_cast((cp >> 18 ) | 0xf0)); + writeOutput(static_cast(((cp >> 12) & 0x3f) | 0x80)); + writeOutput(static_cast(((cp >> 6) & 0x3f) | 0x80)); + writeOutput(static_cast((cp & 0x3f ) | 0x80)); } - return result; } @@ -170,14 +168,13 @@ size_t getUtf8Len(unsigned char ch) template inline -Function utf8ToCodePoint(CharIterator first, CharIterator last, Function f) //f is a unary function taking a CodePoint as single parameter +void utf8ToCodePoint(CharIterator first, CharIterator last, Function writeOutput) //"writeOutput" is a unary function taking a CodePoint { assert_static(sizeof(typename std::iterator_traits::value_type) == 1); - typedef unsigned char Char8; for ( ; first != last; ++first) { - auto getChar = [&](Char8 & ch) -> bool + auto getChar = [&](Char8& ch) -> bool { if (++first == last) { @@ -189,71 +186,55 @@ Function utf8ToCodePoint(CharIterator first, CharIterator last, Function f) //f return true; }; - CodePoint cp = static_cast(*first); - switch (getUtf8Len(static_cast(cp))) + Char8 ch = static_cast(*first); + switch (getUtf8Len(ch)) { case 1: + writeOutput(ch); break; case 2: { - cp = (cp & 0x1f) << 6; - Char8 ch; - if (!getChar(ch)) continue; + CodePoint cp = (ch & 0x1f) << 6; + if (!getChar(ch)) return; cp += ch & 0x3f; + writeOutput(cp); } break; case 3: { - cp = (cp & 0xf) << 12; - Char8 ch; - if (!getChar(ch)) continue; + CodePoint cp = (ch & 0xf) << 12; + if (!getChar(ch)) return; cp += (ch & 0x3f) << 6; - if (!getChar(ch)) continue; + if (!getChar(ch)) return; cp += ch & 0x3f; - + writeOutput(cp); } break; case 4: { - cp = (cp & 0x7) << 18; - Char8 ch; - if (!getChar(ch)) continue; + CodePoint cp = (ch & 0x7) << 18; + if (!getChar(ch)) return; cp += (ch & 0x3f) << 12; - if (!getChar(ch)) continue; + if (!getChar(ch)) return; cp += (ch & 0x3f) << 6; - if (!getChar(ch)) continue; + if (!getChar(ch)) return; cp += ch & 0x3f; + writeOutput(cp); } break; default: assert(false); } - f(cp); } - return f; } -template -class AppendStringIterator: public std::iterator -{ -public: - explicit AppendStringIterator (String& x) : str(&x) {} - AppendStringIterator& operator= (typename String::value_type value) { *str += value; return *this; } - AppendStringIterator& operator* () { return *this; } - AppendStringIterator& operator++ () { return *this; } - AppendStringIterator operator++ (int) { return *this; } -private: - String* str; -}; - - template inline WideString utf8ToWide(const CharString& str, Int2Type<2>) //windows: convert utf8 to utf16 wchar_t { WideString output; utf8ToCodePoint(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf16(cp, AppendStringIterator(output)); }); + [&](CodePoint cp) { codePointToUtf16(cp, [&](Char16 c) { output += static_cast(c); }); }); return output; } @@ -273,7 +254,7 @@ CharString wideToUtf8(const WideString& str, Int2Type<2>) //windows: convert utf { CharString output; utf16ToCodePoint(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf8(cp, AppendStringIterator(output)); }); + [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); return output; } @@ -283,7 +264,7 @@ CharString wideToUtf8(const WideString& str, Int2Type<4>) //other OS: convert ut { CharString output; std::for_each(strBegin(str), strBegin(str) + strLength(str), - [&](CodePoint cp) { codePointToUtf8(cp, AppendStringIterator(output)); }); + [&](CodePoint cp) { codePointToUtf8(cp, [&](char c) { output += c; }); }); return output; } } @@ -292,8 +273,8 @@ CharString wideToUtf8(const WideString& str, Int2Type<4>) //other OS: convert ut template inline WideString utf8ToWide(const CharString& str) { - assert_static((IsSameType::Result, char >::result)); - assert_static((IsSameType::Result, wchar_t>::result)); + assert_static((IsSameType::Type, char >::value)); + assert_static((IsSameType::Type, wchar_t>::value)); return implementation::utf8ToWide(str, Int2Type()); } @@ -302,8 +283,8 @@ WideString utf8ToWide(const CharString& str) template inline CharString wideToUtf8(const WideString& str) { - assert_static((IsSameType::Result, char >::result)); - assert_static((IsSameType::Result, wchar_t>::result)); + assert_static((IsSameType::Type, char >::value)); + assert_static((IsSameType::Type, wchar_t>::value)); return implementation::wideToUtf8(str, Int2Type()); } @@ -326,8 +307,8 @@ template inline TargetString utf8CvrtTo(const SourceString& str) { return utf8CvrtTo(str, - typename GetCharType::Result(), - typename GetCharType::Result()); + typename GetCharType::Type(), + typename GetCharType::Type()); } } -- cgit