Merge branch 'b11.25' into 'master'11.25

add upstream 11.25

See merge request opensource-tracking/FreeFileSync!48

14 files changed, 130 insertions, 90 deletions

diff --git a/Changelog.txt b/Changelog.txt
index ee265eee..f0a3df5c 100644
--- a/Changelog.txt
+++ b/Changelog.txt
@@ -1,5 +1,10 @@
-FreeFileSync 11.25
-------------------
+FreeFileSync 11.25 [2022-08-31]
+-------------------------------
+Fixed crash when normalizing Unicode non-characters
+Fixed crash when accesssing Google Drive
+Fixed regession for decomposed Unicode comparison
+Fixed "exit code 106: --sign is required" error on macOS
+Reset icon cache after each comparison
 
 
 FreeFileSync 11.24 [2022-08-28]
diff --git a/FreeFileSync/Source/afs/ftp.cpp b/FreeFileSync/Source/afs/ftp.cpp
index cd66c047..41d660d4 100644
--- a/FreeFileSync/Source/afs/ftp.cpp
+++ b/FreeFileSync/Source/afs/ftp.cpp
@@ -79,6 +79,8 @@ Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& afs
 
 Zstring ansiToUtfEncoding(const std::string& str) //throw SysError
 {
+    if (str.empty()) return {};
+
     gsize bytesWritten = 0; //not including the terminating null
 
     GError* error = nullptr;
@@ -104,6 +106,8 @@ Zstring ansiToUtfEncoding(const std::string& str) //throw SysError
 
 std::string utfToAnsiEncoding(const Zstring& str) //throw SysError
 {
+    if (str.empty()) return {};
+
     gsize bytesWritten = 0; //not including the terminating null
 
     GError* error = nullptr;
diff --git a/FreeFileSync/Source/afs/gdrive.cpp b/FreeFileSync/Source/afs/gdrive.cpp
index a1b628a7..a220f340 100644
--- a/FreeFileSync/Source/afs/gdrive.cpp
+++ b/FreeFileSync/Source/afs/gdrive.cpp
@@ -2808,7 +2808,7 @@ private:
                 if (!std::equal(std::begin(tmp2), std::end(tmp2), std::begin(DB_FILE_DESCR_OLD)))
                     throw SysError(_("File content is corrupted.") + L" (invalid header)");
 
-                const int version = readNumber<int32_t>(streamIn2);
+                const int version = readNumber<int32_t>(streamIn2); //throw SysErrorUnexpectedEos
                 if (version != 1 && //TODO: remove migration code at some time! 2019-12-05
                     version != 2 && //TODO: remove migration code at some time! 2020-06-11
                     version != 3)   //TODO: remove migration code at some time! 2020-07-03
@@ -2826,7 +2826,7 @@ private:
                 if (!std::equal(std::begin(tmp), std::end(tmp), std::begin(DB_FILE_DESCR)))
                     throw SysError(_("File content is corrupted.") + L" (invalid header)");
 
-                const int version = readNumber<int32_t>(streamIn);
+                const int version = readNumber<int32_t>(streamIn); //throw SysErrorUnexpectedEos
                 if (version != 4 &&
                     version != DB_FILE_VERSION)
                     throw SysError(_("Unsupported data format.") + L' ' + replaceCpy(_("Version: %x"), L"%x", numberTo<std::wstring>(version)));
@@ -2843,6 +2843,7 @@ private:
                     else
                         return makeSharedRef<GdriveDrivesBuffer>(streamInBody, accessBuf.ref()); //throw SysError
                 }();
+
                 return UserSession{accessBuf, drivesBuf};
             }
         }
diff --git a/FreeFileSync/Source/ui/file_grid.cpp b/FreeFileSync/Source/ui/file_grid.cpp
index 6c8a1f4a..824648b2 100644
--- a/FreeFileSync/Source/ui/file_grid.cpp
+++ b/FreeFileSync/Source/ui/file_grid.cpp
@@ -2056,7 +2056,7 @@ void IconManager::startIconUpdater() { assert(iconUpdater_); if (iconUpdater_) i
 
 void filegrid::setupIcons(Grid& gridLeft, Grid& gridCenter, Grid& gridRight, bool showFileIcons, IconBuffer::IconSize sz)
 {
-    auto* provLeft  = dynamic_cast<GridDataLeft*>(gridLeft .getDataProvider());
+    auto* provLeft  = dynamic_cast<GridDataLeft *>(gridLeft .getDataProvider());
     auto* provRight = dynamic_cast<GridDataRight*>(gridRight.getDataProvider());
 
     if (provLeft && provRight)
diff --git a/FreeFileSync/Source/ui/main_dlg.cpp b/FreeFileSync/Source/ui/main_dlg.cpp
index beea4555..82bbc8e5 100644
--- a/FreeFileSync/Source/ui/main_dlg.cpp
+++ b/FreeFileSync/Source/ui/main_dlg.cpp
@@ -313,7 +313,7 @@ void updateTopButton(wxBitmapButton& btn, const wxImage& img, const wxString& va
              stackImages(btnIconImg, btnImg, ImageStackLayout::horizontal, ImageStackAlignment::center, fastFromDIP(5)) :
              stackImages(btnImg, btnIconImg, ImageStackLayout::horizontal, ImageStackAlignment::center, fastFromDIP(5));
 
-    wxSize btnSize = btnImg.GetSize();
+    wxSize btnSize = btnImg.GetSize() + wxSize(fastFromDIP(10), 0) /*border space*/;
     btnSize.x = std::max(btnSize.x, fastFromDIP(TOP_BUTTON_OPTIMAL_WIDTH_DIP));
 
     btnImg = resizeCanvas(btnImg, btnSize, wxALIGN_CENTER);
@@ -411,7 +411,6 @@ void MainDialog::create(const Zstring& globalConfigFilePath,
                         const std::vector<Zstring>& referenceFiles,
                         bool startComparison)
 {
-
     const XmlGlobalSettings globSett = globalSettings ? *globalSettings : tryLoadGlobalConfig(globalConfigFilePath);
 
     try
@@ -4200,6 +4199,9 @@ void MainDialog::onCompare(wxCommandEvent& event)
             updateConfigLastRunStats(std::chrono::system_clock::to_time_t(r.summary.startTime), r.summary.syncResult, getNullPath() /*logFilePath*/);
         }
     }
+
+    //reset icon cache (IconBuffer) after *each* comparison!
+    filegrid::setupIcons(*m_gridMainL, *m_gridMainC, *m_gridMainR, globalCfg_.mainDlg.showIcons, convert(globalCfg_.mainDlg.iconSize));
 }
 
 
diff --git a/FreeFileSync/Source/ui/search_grid.cpp b/FreeFileSync/Source/ui/search_grid.cpp
index cff99d12..37214c05 100644
--- a/FreeFileSync/Source/ui/search_grid.cpp
+++ b/FreeFileSync/Source/ui/search_grid.cpp
@@ -45,7 +45,7 @@ void normalizeForSearch<false /*respectCase*/>(std::wstring& str)
         else if (c == L'\\')
             c = L'/';
         else
-            c = asciiToUpper(c);
+            c = asciiToUpper(c); //caveat, decomposed Unicode form! c might be followed by combining character! Still, should be fine...
 }
 
 
diff --git a/FreeFileSync/Source/version/version.h b/FreeFileSync/Source/version/version.h
index eea2ae14..961e8c9a 100644
--- a/FreeFileSync/Source/version/version.h
+++ b/FreeFileSync/Source/version/version.h
@@ -3,7 +3,7 @@
 
 namespace fff
 {
-const char ffsVersion[] = "11.24"; //internal linkage!
+const char ffsVersion[] = "11.25"; //internal linkage!
 const char FFS_VERSION_SEPARATOR = '.';
 }
 
diff --git a/zen/json.h b/zen/json.h
index 3a9d73f3..be2cfbab 100644
--- a/zen/json.h
+++ b/zen/json.h
@@ -140,7 +140,7 @@ namespace
         {
             UtfDecoder<impl::Char16> decoder(utf16Buf.c_str(), utf16Buf.size());
             while (std::optional<impl::CodePoint> cp = decoder.getNext())
-                impl::codePointToUtf<char>(*cp, [&](char c) { output += c; });
+                codePointToUtf<char>(*cp, [&](char c) { output += c; });
             utf16Buf.clear();
         }
     };
diff --git a/zen/process_exec.cpp b/zen/process_exec.cpp
index df41a627..fb691151 100644
--- a/zen/process_exec.cpp
+++ b/zen/process_exec.cpp
@@ -19,7 +19,7 @@ using namespace zen;
 
 Zstring zen::escapeCommandArg(const Zstring& arg)
 {
-//*INDENT-OFF*
+//*INDENT-OFF*    if not put exactly here, Astyle will seriously mess this .cpp file up!
     Zstring output;
     for (const Zchar c : arg)
         switch (c)
@@ -27,7 +27,7 @@ Zstring zen::escapeCommandArg(const Zstring& arg)
             case  '"': output += "\\\""; break; //Windows: not needed; " cannot be used as file name
             case '\\': output += "\\\\"; break; //Windows: path separator! => don't escape
             case '`':  output += "\\`";  break; //yes, used in some paths => Windows: no escaping required
-            default: output += c; break;
+            default:   output += c; break;
         }
 //*INDENT-ON*
     if (contains(output, Zstr(' ')))
diff --git a/zen/serialize.h b/zen/serialize.h
index b2561808..26202d96 100644
--- a/zen/serialize.h
+++ b/zen/serialize.h
@@ -8,9 +8,6 @@
 #define SERIALIZE_H_839405783574356
 
 #include <functional>
-//#include <cstdint>
-//#include <stdexcept>
-//#include "string_base.h"
 #include "sys_error.h"
 //keep header clean from specific stream implementations! (e.g.file_io.h)! used by abstract.h!
 
@@ -19,36 +16,35 @@ namespace zen
 {
 /* high-performance unformatted serialization (avoiding wxMemoryOutputStream/wxMemoryInputStream inefficiencies)
 
---------------------------
-|Binary Container Concept|
---------------------------
-binary container for data storage: must support "basic" std::vector interface (e.g. std::vector<std::byte>, std::string, Zbase<char>)
+    ----------------------------
+    | Binary Container Concept |
+    ----------------------------
+        binary container for data storage: must support "basic" std::vector interface (e.g. std::vector<std::byte>, std::string, Zbase<char>)
 
+    ---------------------------------
+    | Buffered Input Stream Concept |
+    ---------------------------------
+        struct BufferedInputStream
+        {
+            size_t read(void* buffer, size_t bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream!
+
+            Optional: support stream-copying
+            --------------------------------
+            size_t getBlockSize() const;
+            const IoCallback& notifyUnbufferedIO
+        };
+
+    ----------------------------------
+    | Buffered Output Stream Concept |
+    ----------------------------------
+        struct BufferedOutputStream
+        {
+            void write(const void* buffer, size_t bytesToWrite); //throw X
 
--------------------------------
-|Buffered Input Stream Concept|
--------------------------------
-struct BufferedInputStream
-{
-    size_t read(void* buffer, size_t bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream!
-
-Optional: support stream-copying
---------------------------------
-    size_t getBlockSize() const;
-    const IoCallback& notifyUnbufferedIO
-};
-
---------------------------------
-|Buffered Output Stream Concept|
---------------------------------
-struct BufferedOutputStream
-{
-    void write(const void* buffer, size_t bytesToWrite); //throw X
-
-Optional: support stream-copying
---------------------------------
-    const IoCallback& notifyUnbufferedIO
-};                                                                           */
+            Optional: support stream-copying
+            --------------------------------
+            const IoCallback& notifyUnbufferedIO
+        };                                                                           */
 using IoCallback = std::function<void(int64_t bytesDelta)>; //throw X
 
 
@@ -116,6 +112,7 @@ private:
     size_t pos_ = 0;
 };
 
+
 template <class BinContainer>
 struct MemoryStreamOut
 {
@@ -144,9 +141,6 @@ private:
 
 
 
-
-
-
 //-----------------------implementation-------------------------------
 template <class BufferedInputStream, class BufferedOutputStream> inline
 void bufferedStreamCopy(BufferedInputStream& streamIn,   //throw X
@@ -214,10 +208,13 @@ void writeNumber(BufferedOutputStream& stream, const N& num)
 template <class C, class BufferedOutputStream> inline
 void writeContainer(BufferedOutputStream& stream, const C& cont) //don't even consider UTF8 conversions here, we're handling arbitrary binary data!
 {
-    const auto len = cont.size();
-    writeNumber(stream, static_cast<uint32_t>(len));
-    if (len > 0)
-        writeArray(stream, &cont[0], sizeof(typename C::value_type) * len); //don't use c_str(), but access uniformly via STL interface
+    const auto size = cont.size();
+
+    assert(size <= INT32_MAX);
+    writeNumber(stream, static_cast<int32_t>(size)); //use *signed* integer to help catch data corruption
+
+    if (size > 0)
+        writeArray(stream, &cont[0], sizeof(typename C::value_type) * size); //don't use c_str(), but access uniformly via STL interface
 }
 
 
@@ -244,18 +241,21 @@ N readNumber(BufferedInputStream& stream) //throw SysErrorUnexpectedEos
 template <class C, class BufferedInputStream> inline
 C readContainer(BufferedInputStream& stream) //throw SysErrorUnexpectedEos
 {
+    const auto size = readNumber<int32_t>(stream); //throw SysErrorUnexpectedEos
+    if (size < 0) //most likely due to data corruption!
+        throw SysErrorUnexpectedEos();
+
     C cont;
-    auto strLength = readNumber<uint32_t>(stream); //throw SysErrorUnexpectedEos
-    if (strLength > 0)
+    if (size > 0)
     {
         try
         {
-            cont.resize(strLength); //throw std::length_error, std::bad_alloc
+            cont.resize(size); //throw std::length_error, std::bad_alloc
         }
-        catch (std::length_error&) { throw SysErrorUnexpectedEos(); } //most likely this is due to data corruption!
+        catch (std::length_error&) { throw SysErrorUnexpectedEos(); } //most likely due to data corruption!
         catch (   std::bad_alloc&) { throw SysErrorUnexpectedEos(); } //
 
-        readArray(stream, &cont[0], sizeof(typename C::value_type) * strLength); //throw SysErrorUnexpectedEos
+        readArray(stream, &cont[0], sizeof(typename C::value_type) * size); //throw SysErrorUnexpectedEos
     }
     return cont;
 }
diff --git a/zen/string_tools.h b/zen/string_tools.h
index cafff3d5..181a3951 100644
--- a/zen/string_tools.h
+++ b/zen/string_tools.h
@@ -263,7 +263,7 @@ bool equalString(const S& lhs, const T& rhs)
 template <class S, class T> inline
 bool equalAsciiNoCase(const S& lhs, const T& rhs)
 {
-    assert(isAsciiString(lhs) || isAsciiString(rhs));
+    //assert(isAsciiString(lhs) || isAsciiString(rhs));
     const size_t lhsLen = strLength(lhs);
     return lhsLen == strLength(rhs) && impl::strcmpAsciiNoCase(strBegin(lhs), strBegin(rhs), lhsLen) == std::weak_ordering::equivalent;
 }
diff --git a/zen/utf.h b/zen/utf.h
index ca231602..56b1ff55 100644
--- a/zen/utf.h
+++ b/zen/utf.h
@@ -222,15 +222,9 @@ private:
 
 //----------------------------------------------------------------------------------------------------------------
 
-template <class Function> inline void codePointToUtf(CodePoint cp, Function writeOutput, std::integral_constant<int, 1>) { codePointToUtf8 (cp, writeOutput); } //UTF8-char
-template <class Function> inline void codePointToUtf(CodePoint cp, Function writeOutput, std::integral_constant<int, 2>) { codePointToUtf16(cp, writeOutput); } //Windows: UTF16-wchar_t
-template <class Function> inline void codePointToUtf(CodePoint cp, Function writeOutput, std::integral_constant<int, 4>) { writeOutput(cp); } //other OS: UTF32-wchar_t
-
-template <class CharType, class Function> inline
-void codePointToUtf(CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a CharType
-{
-    return codePointToUtf(cp, writeOutput, std::integral_constant<int, sizeof(CharType)>());
-}
+template <class Function> inline void codePointToUtfImpl(CodePoint cp, Function writeOutput, std::integral_constant<int, 1>) { codePointToUtf8 (cp, writeOutput); } //UTF8-char
+template <class Function> inline void codePointToUtfImpl(CodePoint cp, Function writeOutput, std::integral_constant<int, 2>) { codePointToUtf16(cp, writeOutput); } //Windows: UTF16-wchar_t
+template <class Function> inline void codePointToUtfImpl(CodePoint cp, Function writeOutput, std::integral_constant<int, 4>) { writeOutput(cp); } //other OS: UTF32-wchar_t
 
 //----------------------------------------------------------------------------------------------------------------
 
@@ -277,9 +271,18 @@ private:
 };
 }
 
+
 template <class CharType>
 using UtfDecoder = impl::UtfDecoderImpl<CharType, sizeof(CharType)>;
 
+
+template <class CharType, class Function> inline
+void codePointToUtf(impl::CodePoint cp, Function writeOutput) //"writeOutput" is a unary function taking a CharType
+{
+    return impl::codePointToUtfImpl(cp, writeOutput, std::integral_constant<int, sizeof(CharType)>());
+}
+
+
 //-------------------------------------------------------------------------------------------
 
 template <class UtfString> inline
diff --git a/zen/zstring.cpp b/zen/zstring.cpp
index 1e29e461..3f5328f7 100644
--- a/zen/zstring.cpp
+++ b/zen/zstring.cpp
@@ -15,7 +15,7 @@ Zstring getUnicodeNormalFormNonAscii(const Zstring& str)
 {
     //Example: const char* decomposed  = "\x6f\xcc\x81";
     //         const char* precomposed = "\xc3\xb3";
-    assert(!isAsciiString(str));
+    assert(!isAsciiString(str)); //includes "not-empty" check
     assert(str.find(Zchar('\0')) == Zstring::npos); //don't expect embedded nulls!
 
     try
@@ -51,14 +51,14 @@ Zstring getUpperCaseNonAscii(const Zstring& str)
     Zstring strNorm = getUnicodeNormalFormNonAscii(str);
     try
     {
-        static_assert(sizeof(impl::CodePoint) == sizeof(gunichar));
         Zstring output;
         output.reserve(strNorm.size());
 
         UtfDecoder<char> decoder(strNorm.c_str(), strNorm.size());
         while (const std::optional<impl::CodePoint> cp = decoder.getNext())
-            impl::codePointToUtf<char>(::g_unichar_toupper(*cp), [&](char c) { output += c; }); //don't use std::towupper: *incomplete* and locale-dependent!
+            codePointToUtf<char>(::g_unichar_toupper(*cp), [&](char c) { output += c; }); //don't use std::towupper: *incomplete* and locale-dependent!
 
+        static_assert(sizeof(impl::CodePoint) == sizeof(gunichar));
         return output;
 
     }
@@ -89,6 +89,10 @@ namespace
 {
 std::weak_ordering compareNoCaseUtf8(const char* lhs, size_t lhsLen, const char* rhs, size_t rhsLen)
 {
+    //expect Unicode normalized strings!
+    assert(std::string(lhs, lhsLen) == getUnicodeNormalForm(std::string(lhs, lhsLen)));
+    assert(std::string(rhs, rhsLen) == getUnicodeNormalForm(std::string(rhs, rhsLen)));
+
     //- strncasecmp implements ASCII CI-comparsion only! => signature is broken for UTF8-input; toupper() similarly doesn't support Unicode
     //- wcsncasecmp: https://opensource.apple.com/source/Libc/Libc-763.12/string/wcsncasecmp-fbsd.c
     // => re-implement comparison based on g_unichar_tolower() to avoid memory allocations
@@ -103,12 +107,13 @@ std::weak_ordering compareNoCaseUtf8(const char* lhs, size_t lhsLen, const char*
             return !cpR <=> !cpL;
 
         static_assert(sizeof(gunichar) == sizeof(impl::CodePoint));
+        static_assert(std::is_unsigned_v<gunichar>, "unsigned char-comparison is the convention!");
 
         //ordering: "to lower" converts to higher code points than "to upper"
         const gunichar charL = ::g_unichar_toupper(*cpL); //note: tolower can be ambiguous, so don't use:
         const gunichar charR = ::g_unichar_toupper(*cpR); //e.g. "Σ" (upper case) can be lower-case "ς" in the end of the word or "σ" in the middle.
         if (charL != charR)
-            return makeUnsigned(charL) <=> makeUnsigned(charR); //unsigned char-comparison is the convention!
+            return charL <=> charR;
     }
 }
 }
@@ -206,25 +211,48 @@ std::weak_ordering compareNatural(const Zstring& lhs, const Zstring& rhs)
 
 std::weak_ordering compareNoCase(const Zstring& lhs, const Zstring& rhs)
 {
-    //fast path: no need for extra memory allocations => ~ 6x speedup
-    const size_t minSize = std::min(lhs.size(), rhs.size());
+    //fast path: no memory allocations => ~ 6x speedup
+    if (isAsciiString(lhs) && isAsciiString(rhs))
+    {
+        const size_t minSize = std::min(lhs.size(), rhs.size());
+        for (size_t i = 0; i < minSize; ++i)
+        {
+            //ordering: do NOT call compareAsciiNoCase(), which uses asciiToLower()!
+            const Zchar lUp = asciiToUpper(lhs[i]); //
+            const Zchar rUp = asciiToUpper(rhs[i]); //no surprises: emulate getUpperCase() [verified!]
+            if (lUp != rUp)                         //
+                return lUp <=> rUp;                 //
+        }
+        return lhs.size() <=> rhs.size();
+    }
+    //--------------------------------------
+
+    //can't we instead skip isAsciiString() and compare chars as long as isAsciiChar()?
+    // => NOPE! e.g. decomposed Unicode! A seemingly single isAsciiChar() might be followed by a combining character!!!
+
+    return getUpperCase(lhs) <=> getUpperCase(rhs);
+}
+
+
+bool equalNoCase(const Zstring& lhs, const Zstring& rhs)
+{
+    //fast-path: no need for extra memory allocations
+    const bool isAsciiL = isAsciiString(lhs);
+    const bool isAsciiR = isAsciiString(rhs);
+    if (isAsciiL != isAsciiR)
+        return false;
 
-    size_t i = 0;
-    for (; i < minSize; ++i)
+    if (isAsciiL)
     {
-        const Zchar l = lhs[i];
-        const Zchar r = rhs[i];
-        if (!isAsciiChar(l) || !isAsciiChar(r))
-            goto slowPath; //=> let's NOT make assumptions how getUpperCase() compares "ASCII <=> non-ASCII"
-
-        const Zchar lUp = asciiToUpper(l); //
-        const Zchar rUp = asciiToUpper(r); //no surprises: emulate getUpperCase() [verified!]
-        if (lUp != rUp)                    //
-            return lUp <=> rUp;            //
+        if (lhs.size() != rhs.size())
+            return false;
+
+        for (size_t i = 0; i < lhs.size(); ++i)
+            if (asciiToUpper(lhs[i]) !=
+                asciiToUpper(rhs[i]))
+                return false;
+        return true;
     }
-    return lhs.size() <=> rhs.size();
-slowPath: //--------------------------------------
 
-    return compareNoCaseUtf8(lhs.c_str() + i, lhs.size() - i, 
-                             rhs.c_str() + i, rhs.size() - i);
+    return getUpperCaseNonAscii(lhs) == getUpperCaseNonAscii(rhs);
 }
diff --git a/zen/zstring.h b/zen/zstring.h
index 70b9f448..692217c1 100644
--- a/zen/zstring.h
+++ b/zen/zstring.h
@@ -63,10 +63,7 @@ template<> struct std::hash<ZstringNoCase> { size_t operator()(const ZstringNoCa
 
 std::weak_ordering compareNoCase(const Zstring& lhs, const Zstring& rhs);
 
-inline
-bool equalNoCase(const Zstring& lhs, const Zstring& rhs) { return compareNoCase(lhs, rhs) == std::weak_ordering::equivalent;  }
-//note: the "lhs.size() != rhs.size()" short-cut would require two isAsciiString() checks
-//=> generally SLOWER than starting comparison directly during first pass and breaking on first difference!
+bool equalNoCase(const Zstring& lhs, const Zstring& rhs);
 
 //------------------------------------------------------------------------------------------
 std::weak_ordering compareNatural(const Zstring& lhs, const Zstring& rhs);