1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
// *****************************************************************************
// * This file is part of the FreeFileSync project. It is distributed under *
// * GNU General Public License: https://www.gnu.org/licenses/gpl-3.0 *
// * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved *
// *****************************************************************************
#include "zstring.h"
#include <glib.h>
#include "sys_error.h"
using namespace zen;
Zstring getUnicodeNormalForm(const Zstring& str)
{
//fast pre-check:
if (isAsciiString(str)) //perf: in the range of 3.5ns
return str;
static_assert(std::is_same_v<decltype(str), const Zbase<Zchar>&>, "god bless our ref-counting! => save output string memory consumption!");
//Example: const char* decomposed = "\x6f\xcc\x81";
// const char* precomposed = "\xc3\xb3";
try
{
gchar* outStr = ::g_utf8_normalize(str.c_str(), str.length(), G_NORMALIZE_DEFAULT_COMPOSE);
if (!outStr)
throw SysError(formatSystemError("g_utf8_normalize(" + utfTo<std::string>(str) + ')', L"", L"Conversion failed."));
ZEN_ON_SCOPE_EXIT(::g_free(outStr));
return outStr;
}
catch ([[maybe_unused]] const SysError& e)
{
assert(false);
return str;
}
}
Zstring getUpperCase(const Zstring& str)
{
assert(str.find(Zchar('\0')) == Zstring::npos); //don't expect embedded nulls!
//fast pre-check:
if (isAsciiString(str)) //perf: in the range of 3.5ns
{
Zstring output = str;
for (Zchar& c : output)
c = asciiToUpper(c);
return output;
}
Zstring strNorm = getUnicodeNormalForm(str);
try
{
static_assert(sizeof(impl::CodePoint) == sizeof(gunichar));
Zstring output;
output.reserve(strNorm.size());
UtfDecoder<char> decoder(strNorm.c_str(), strNorm.size());
while (const std::optional<impl::CodePoint> cp = decoder.getNext())
impl::codePointToUtf<char>(::g_unichar_toupper(*cp), [&](char c) { output += c; }); //don't use std::towupper: *incomplete* and locale-dependent!
return output;
}
catch (SysError&)
{
assert(false);
return str;
}
}
namespace
{
std::weak_ordering compareNoCaseUtf8(const char* lhs, size_t lhsLen, const char* rhs, size_t rhsLen)
{
//- strncasecmp implements ASCII CI-comparsion only! => signature is broken for UTF8-input; toupper() similarly doesn't support Unicode
//- wcsncasecmp: https://opensource.apple.com/source/Libc/Libc-763.12/string/wcsncasecmp-fbsd.c
// => re-implement comparison based on g_unichar_tolower() to avoid memory allocations
UtfDecoder<char> decL(lhs, lhsLen);
UtfDecoder<char> decR(rhs, rhsLen);
for (;;)
{
const std::optional<impl::CodePoint> cpL = decL.getNext();
const std::optional<impl::CodePoint> cpR = decR.getNext();
if (!cpL || !cpR)
return !cpR <=> !cpL;
static_assert(sizeof(gunichar) == sizeof(impl::CodePoint));
const gunichar charL = ::g_unichar_toupper(*cpL); //note: tolower can be ambiguous, so don't use:
const gunichar charR = ::g_unichar_toupper(*cpR); //e.g. "Σ" (upper case) can be lower-case "ς" in the end of the word or "σ" in the middle.
if (charL != charR)
//ordering: "to lower" converts to higher code points than "to upper"
return makeUnsigned(charL) <=> makeUnsigned(charR); //unsigned char-comparison is the convention!
}
}
}
std::weak_ordering compareNatural(const Zstring& lhs, const Zstring& rhs)
{
/* Unicode normal forms:
Windows: CompareString() already ignores NFD/NFC differences: nice...
Linux: g_unichar_toupper() can't ignore differences
macOS: CFStringCompare() considers differences */
const Zstring& lhsNorm = getUnicodeNormalForm(lhs);
const Zstring& rhsNorm = getUnicodeNormalForm(rhs);
const char* strL = lhsNorm.c_str();
const char* strR = rhsNorm.c_str();
const char* const strEndL = strL + lhsNorm.size();
const char* const strEndR = strR + rhsNorm.size();
/* - compare strings after conceptually creating blocks of whitespace/numbers/text
- implement strict weak ordering!
- don't follow broken "strnatcasecmp": https://github.com/php/php-src/blob/master/ext/standard/strnatcmp.c
1. incorrect non-ASCII CI-comparison
2. incorrect bounds checks
3. incorrect trimming of *all* whitespace
4. arbitrary handling of leading 0 only at string begin
5. incorrect handling of whitespace following a number
6. code is a mess */
for (;;)
{
if (strL == strEndL || strR == strEndR)
return (strL != strEndL) <=> (strR != strEndR); //"nothing" before "something"
//note: "something" never would have been condensed to "nothing" further below => can finish evaluation here
const bool wsL = isWhiteSpace(*strL);
const bool wsR = isWhiteSpace(*strR);
if (wsL != wsR)
return !wsL <=> !wsR; //whitespace before non-ws!
if (wsL)
{
++strL, ++strR;
while (strL != strEndL && isWhiteSpace(*strL)) ++strL;
while (strR != strEndR && isWhiteSpace(*strR)) ++strR;
continue;
}
const bool digitL = isDigit(*strL);
const bool digitR = isDigit(*strR);
if (digitL != digitR)
return !digitL <=> !digitR; //numbers before chars!
if (digitL)
{
while (strL != strEndL && *strL == '0') ++strL;
while (strR != strEndR && *strR == '0') ++strR;
int rv = 0;
for (;; ++strL, ++strR)
{
const bool endL = strL == strEndL || !isDigit(*strL);
const bool endR = strR == strEndR || !isDigit(*strR);
if (endL != endR)
return !endL <=> !endR; //more digits means bigger number
if (endL)
break; //same number of digits
if (rv == 0 && *strL != *strR)
rv = *strL - *strR; //found first digit difference comparing from left
}
if (rv != 0)
return rv <=> 0;
continue;
}
//compare full junks of text: consider unicode encoding!
const char* textBeginL = strL++;
const char* textBeginR = strR++; //current char is neither white space nor digit at this point!
while (strL != strEndL && !isWhiteSpace(*strL) && !isDigit(*strL)) ++strL;
while (strR != strEndR && !isWhiteSpace(*strR) && !isDigit(*strR)) ++strR;
if (const std::weak_ordering cmp = compareNoCaseUtf8(textBeginL, strL - textBeginL, textBeginR, strR - textBeginR);
cmp != std::weak_ordering::equivalent)
return cmp;
}
}
|
