8.7

1 files changed, 262 insertions, 148 deletions

diff --git a/wx+/http.cpp b/wx+/http.cpp
index ce3de482..3428546e 100644
--- a/wx+/http.cpp
+++ b/wx+/http.cpp
@@ -31,186 +31,259 @@ namespace
     #endif
 #endif
 
+struct UrlRedirectError
+{
+    UrlRedirectError(const std::wstring& url) : newUrl(url) {}
+    std::wstring newUrl;
+};
+}
 
-std::string sendHttpRequestImpl(const std::wstring& url, //throw SysError
-                                const std::wstring& userAgent,
-                                const std::string* postParams, //issue POST if bound, GET otherwise
-                                int level = 0)
+
+class HttpInputStream::Impl
 {
-    assert(!startsWith(makeUpperCopy(url), L"HTTPS:")); //not supported by wxHTTP!
-    const std::wstring urlFmt = startsWith(makeUpperCopy(url), L"HTTP://") ? afterFirst(url, L"://", IF_MISSING_RETURN_NONE) : url;
-    const std::wstring server =       beforeFirst(urlFmt, L'/', IF_MISSING_RETURN_ALL);
-    const std::wstring page   = L'/' + afterFirst(urlFmt, L'/', IF_MISSING_RETURN_NONE);
+public:
+    Impl(const std::wstring& url, const std::wstring& userAgent, //throw SysError, UrlRedirectError
+         const std::string* postParams) //issue POST if bound, GET otherwise
+    {
+        ZEN_ON_SCOPE_FAIL( cleanup(); /*destructor call would lead to member double clean-up!!!*/ );
+
+        assert(!startsWith(makeUpperCopy(url), L"HTTPS:")); //not supported by wxHTTP!
+        const std::wstring urlFmt = startsWith(makeUpperCopy(url), L"HTTP://") ||
+                                    startsWith(makeUpperCopy(url), L"HTTPS://") ? afterFirst(url, L"://", IF_MISSING_RETURN_NONE) : url;
+        const std::wstring server =       beforeFirst(urlFmt, L'/', IF_MISSING_RETURN_ALL);
+        const std::wstring page   = L'/' + afterFirst(urlFmt, L'/', IF_MISSING_RETURN_NONE);
 
 #ifdef ZEN_WIN
-    //WinInet: 1. uses IE proxy settings! :) 2. follows HTTP redirects by default 3. swallows HTTPS if needed
-    HINTERNET hInternet = ::InternetOpen(userAgent.c_str(),            //_In_  LPCTSTR lpszAgent,
-                                         INTERNET_OPEN_TYPE_PRECONFIG, //_In_  DWORD dwAccessType,
-                                         nullptr, //_In_  LPCTSTR lpszProxyName,
-                                         nullptr, //_In_  LPCTSTR lpszProxyBypass,
-                                         0);      //_In_  DWORD dwFlags
-    if (!hInternet)
-        THROW_LAST_SYS_ERROR(L"InternetOpen");
-    ZEN_ON_SCOPE_EXIT(::InternetCloseHandle(hInternet));
+        //WinInet: 1. uses IE proxy settings! :) 2. follows HTTP redirects by default 3. swallows HTTPS if needed
+        hInternet_ = ::InternetOpen(userAgent.c_str(),            //_In_  LPCTSTR lpszAgent,
+                                    INTERNET_OPEN_TYPE_PRECONFIG, //_In_  DWORD dwAccessType,
+                                    nullptr, //_In_  LPCTSTR lpszProxyName,
+                                    nullptr, //_In_  LPCTSTR lpszProxyBypass,
+                                    0);      //_In_  DWORD dwFlags
+        if (!hInternet_)
+            THROW_LAST_SYS_ERROR(L"InternetOpen");
+
+        hSession_ = ::InternetConnect(hInternet_,     //_In_ HINTERNET     hInternet,
+                                      server.c_str(), //_In_ LPCTSTR       lpszServerName,
+                                      INTERNET_DEFAULT_HTTP_PORT, //_In_ INTERNET_PORT nServerPort,
+                                      nullptr,        //_In_ LPCTSTR       lpszUsername,
+                                      nullptr,        //_In_ LPCTSTR       lpszPassword,
+                                      INTERNET_SERVICE_HTTP, //_In_ DWORD         dwService,
+                                      0,              //_In_ DWORD         dwFlags,
+                                      0);             //_In_ DWORD_PTR     dwContext
+        if (!hSession_)
+            THROW_LAST_SYS_ERROR(L"InternetConnect");
+
+        const wchar_t* acceptTypes[] = { L"*/*", nullptr };
+        DWORD requestFlags =
+            //INTERNET_FLAG_KEEP_CONNECTION |
+            //      the combination 1. INTERNET_FLAG_KEEP_CONNECTION (= adds "Connection: Keep-Alive" but NOT "Keep-Alive: timeout" to the header)
+            //                      2. *no* "Keep-Alive: timeout" header entry 3. call from within VM and 4. *no* Fiddler running 5. HTTP POST
+            //      leads to Godaddy blocking the IP: http://www.freefilesync.org/forum/viewtopic.php?t=3855
+            //      => it seems a broken keep alive header is the trigger: But why is it then working outside the VM or when Fiddler is running??? Why not a problem for HTTP GET?
+            //          note: HTTP/1.1 has keep-alive semantics by default, so this flag is probably useless anyway
+            INTERNET_FLAG_NO_UI;
+
+        if (postParams)
+        {
+            requestFlags |= INTERNET_FLAG_NO_AUTO_REDIRECT; //POST would be re-issued as GET during auto-redirect => handle ourselves!
+        }
+        else //HTTP GET
+        {
+            requestFlags |= INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP;
+            requestFlags |= INTERNET_FLAG_RELOAD; //not relevant for POST (= never cached)
+        }
 
-    HINTERNET hSession = ::InternetConnect(hInternet,      //_In_ HINTERNET     hInternet,
-                                           server.c_str(), //_In_ LPCTSTR       lpszServerName,
-                                           INTERNET_DEFAULT_HTTP_PORT, //_In_ INTERNET_PORT nServerPort,
-                                           nullptr,        //_In_ LPCTSTR       lpszUsername,
-                                           nullptr,        //_In_ LPCTSTR       lpszPassword,
-                                           INTERNET_SERVICE_HTTP, //_In_ DWORD         dwService,
-                                           0,              //_In_ DWORD         dwFlags,
-                                           0);             //_In_ DWORD_PTR     dwContext
-    if (!hSession)
-        THROW_LAST_SYS_ERROR(L"InternetConnect");
-    ZEN_ON_SCOPE_EXIT(::InternetCloseHandle(hSession));
-
-    const wchar_t* acceptTypes[] = { L"*/*", nullptr };
-    DWORD requestFlags = 
-		//INTERNET_FLAG_KEEP_CONNECTION |
-		//		the combination 1. INTERNET_FLAG_KEEP_CONNECTION (= adds "Connection: Keep-Alive" but NOT "Keep-Alive: timeout" to the header)
-		//						2. *no* "Keep-Alive: timeout" header entry 3. call from within VM and 4. *no* Fiddler running 5. HTTP POST
-		//		leads to Godaddy blocking the IP: http://www.freefilesync.org/forum/viewtopic.php?t=3855
-		//		=> it seems a broken keep alive header is the trigger: But why is it then working outside the VM or when Fiddler is running??? Why not a problem for HTTP GET?
-		//			note: HTTP/1.1 has keep-alive semantics by default, so this flag is probably useless anyway
-                         INTERNET_FLAG_NO_UI;
-
-    if (postParams)
-	{
-		requestFlags |= INTERNET_FLAG_NO_AUTO_REDIRECT; //POST would be re-issued as GET during auto-redirect => handle ourselves!
-	}
-	else //HTTP GET
-	{
-		requestFlags |= INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP;
-		requestFlags |= INTERNET_FLAG_RELOAD; //not relevant for POST (= never cached)
-	}
-
-    HINTERNET hRequest = ::HttpOpenRequest(hSession,     //_In_ HINTERNET hConnect,
-                                           postParams ? L"POST" : L"GET", //_In_ LPCTSTR   lpszVerb,
-                                           page.c_str(), //_In_ LPCTSTR   lpszObjectName,
-                                           nullptr,      //_In_ LPCTSTR   lpszVersion,
-                                           nullptr,      //_In_ LPCTSTR   lpszReferer,
-                                           acceptTypes,  //_In_ LPCTSTR   *lplpszAcceptTypes,
-                                           requestFlags, //_In_ DWORD     dwFlags,
-                                           0); //_In_ DWORD_PTR dwContext
-    if (!hRequest)
-        THROW_LAST_SYS_ERROR(L"HttpOpenRequest");
-    ZEN_ON_SCOPE_EXIT(::InternetCloseHandle(hRequest));
+        hRequest_ = ::HttpOpenRequest(hSession_,    //_In_ HINTERNET hConnect,
+                                      postParams ? L"POST" : L"GET", //_In_ LPCTSTR   lpszVerb,
+                                      page.c_str(), //_In_ LPCTSTR   lpszObjectName,
+                                      nullptr,      //_In_ LPCTSTR   lpszVersion,
+                                      nullptr,      //_In_ LPCTSTR   lpszReferer,
+                                      acceptTypes,  //_In_ LPCTSTR   *lplpszAcceptTypes,
+                                      requestFlags, //_In_ DWORD     dwFlags,
+                                      0); //_In_ DWORD_PTR dwContext
+        if (!hRequest_)
+            THROW_LAST_SYS_ERROR(L"HttpOpenRequest");
 
-     const std::wstring headers = postParams ? L"Content-type: application/x-www-form-urlencoded" : L"";
+        const std::wstring headers = postParams ? L"Content-type: application/x-www-form-urlencoded" : L"";
 
-	assert(std::all_of(headers.begin(), headers.end(), [](wchar_t c){ return makeUnsigned(c) < 128; }));
-	//HttpSendRequest has finicky behavior for non-ASCII headers: https://msdn.microsoft.com/en-us/library/windows/desktop/aa384247
+        assert(std::all_of(headers.begin(), headers.end(), [](wchar_t c) { return makeUnsigned(c) < 128; }));
+        //HttpSendRequest has finicky behavior for non-ASCII headers: https://msdn.microsoft.com/en-us/library/windows/desktop/aa384247
 
-    std::string postParamsBuf = postParams ? *postParams : "";
+        std::string postParamsBuf = postParams ? *postParams : "";
 
-    if (!::HttpSendRequest(hRequest,              //_In_ HINTERNET hRequest,
-                           headers.c_str(),       //_In_ LPCTSTR   lpszHeaders,
-                           static_cast<DWORD>(headers.size()),                  //_In_ DWORD     dwHeadersLength,
-                           postParamsBuf.empty() ? nullptr : &postParamsBuf[0], //_In_ LPVOID    lpOptional,
-                           static_cast<DWORD>(postParamsBuf.size())))           //_In_ DWORD     dwOptionalLength
-        THROW_LAST_SYS_ERROR(L"HttpSendRequest");
+        if (!::HttpSendRequest(hRequest_,             //_In_ HINTERNET hRequest,
+                               headers.c_str(),       //_In_ LPCTSTR   lpszHeaders,
+                               static_cast<DWORD>(headers.size()),                  //_In_ DWORD     dwHeadersLength,
+                               postParamsBuf.empty() ? nullptr : &postParamsBuf[0], //_In_ LPVOID    lpOptional,
+                               static_cast<DWORD>(postParamsBuf.size())))           //_In_ DWORD     dwOptionalLength
+            THROW_LAST_SYS_ERROR(L"HttpSendRequest");
 
-    DWORD sc = 0;
-    {
-        DWORD bufLen = sizeof(sc);
-        if (!::HttpQueryInfo(hRequest, //_In_     HINTERNET hRequest,
-                             HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, //_In_     DWORD dwInfoLevel,
-                             &sc,      //_Inout_  LPVOID lpvBuffer,
-                             &bufLen,  //_Inout_  LPDWORD lpdwBufferLength,
-                             nullptr)) //_Inout_  LPDWORD lpdwIndex
-            THROW_LAST_SYS_ERROR(L"HttpQueryInfo: HTTP_QUERY_STATUS_CODE");
-    }
+        DWORD sc = 0;
+        {
+            DWORD bufLen = sizeof(sc);
+            if (!::HttpQueryInfo(hRequest_, //_In_     HINTERNET hRequest,
+                                 HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, //_In_     DWORD dwInfoLevel,
+                                 &sc,       //_Inout_  LPVOID lpvBuffer,
+                                 &bufLen,   //_Inout_  LPDWORD lpdwBufferLength,
+                                 nullptr))  //_Inout_  LPDWORD lpdwIndex
+                THROW_LAST_SYS_ERROR(L"HttpQueryInfo: HTTP_QUERY_STATUS_CODE");
+        }
 
-    //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
-    if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
-    {
-        if (level < 5) //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop."
+        //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
+        if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
         {
             DWORD bufLen = 10000;
             std::wstring location(bufLen, L'\0');
-            if (!::HttpQueryInfo(hRequest, HTTP_QUERY_LOCATION, &*location.begin(), &bufLen, nullptr))
+            if (!::HttpQueryInfo(hRequest_, HTTP_QUERY_LOCATION, &*location.begin(), &bufLen, nullptr))
                 THROW_LAST_SYS_ERROR(L"HttpQueryInfo: HTTP_QUERY_LOCATION");
             if (bufLen >= location.size()) //HttpQueryInfo expected to write terminating zero
                 throw SysError(L"HttpQueryInfo: HTTP_QUERY_LOCATION, buffer overflow");
             location.resize(bufLen);
 
-            if (!location.empty())
-                return sendHttpRequestImpl(location, userAgent, postParams, level + 1);
+            if (location.empty())
+                throw SysError(L"Unresolvable redirect. Empty target Location.");
+
+            throw UrlRedirectError(location);
+        }
+
+        if (sc != HTTP_STATUS_OK) //200
+            throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));
+        //e.g. 404 - HTTP_STATUS_NOT_FOUND
+
+#else
+        assert(std::this_thread::get_id() == mainThreadId);
+        assert(wxApp::IsMainLoopRunning());
+
+        webAccess_.SetHeader(L"User-Agent", userAgent);
+        webAccess_.SetTimeout(10 /*[s]*/); //default: 10 minutes: WTF are these wxWidgets people thinking???
+
+        if (!webAccess_.Connect(server)) //will *not* fail for non-reachable url here!
+            throw SysError(L"wxHTTP::Connect");
+
+        if (postParams)
+            if (!webAccess_.SetPostText(L"application/x-www-form-urlencoded", utfCvrtTo<wxString>(*postParams)))
+                throw SysError(L"wxHTTP::SetPostText");
+
+        httpStream_.reset(webAccess_.GetInputStream(page)); //pass ownership
+        const int sc = webAccess_.GetResponse();
+
+        //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
+        if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
+        {
+            const std::wstring newUrl(webAccess_.GetHeader(L"Location"));
+            if (newUrl.empty())
+                throw SysError(L"Unresolvable redirect. Empty target Location.");
+
+            throw UrlRedirectError(newUrl);
         }
-        throw SysError(L"Unresolvable redirect.");
+
+        if (sc != 200) //HTTP_STATUS_OK
+            throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));
+
+        if (!httpStream_ || webAccess_.GetError() != wxPROTO_NOERR)
+            throw SysError(L"wxHTTP::GetError (" + numberTo<std::wstring>(webAccess_.GetError()) + L")");
+#endif
     }
 
-    if (sc != HTTP_STATUS_OK) //200
-        throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));
-    //e.g. 404 - HTTP_STATUS_NOT_FOUND
+    ~Impl() { cleanup(); }
 
-    std::string buffer;
-    const DWORD blockSize = 64 * 1024;
-    //internet says "HttpQueryInfo() + HTTP_QUERY_CONTENT_LENGTH" not supported by all http servers...
-    for (;;)
+    size_t tryRead(void* buffer, size_t bytesToRead) //throw SysError; may return short, only 0 means EOF!
     {
-        buffer.resize(buffer.size() + blockSize);
+        if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check!
+            throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));
 
+#ifdef ZEN_WIN
+        //"HttpQueryInfo() + HTTP_QUERY_CONTENT_LENGTH" not supported by all http servers...
         DWORD bytesRead = 0;
-        if (!::InternetReadFile(hRequest,    //_In_   HINTERNET hFile,
-                                &*(buffer.begin() + buffer.size() - blockSize),  //_Out_  LPVOID lpBuffer,
-                                blockSize,   //_In_   DWORD dwNumberOfBytesToRead,
+        if (!::InternetReadFile(hRequest_,   //_In_   HINTERNET hFile,
+                                buffer,      //_Out_  LPVOID lpBuffer,
+                                static_cast<DWORD>(bytesToRead), //_In_   DWORD dwNumberOfBytesToRead,
                                 &bytesRead)) //_Out_  LPDWORD lpdwNumberOfBytesRead
             THROW_LAST_SYS_ERROR(L"InternetReadFile");
+#else
+        httpStream_->Read(buffer, bytesToRead);
 
-        if (bytesRead > blockSize) //better safe than sorry
+        const wxStreamError ec = httpStream_->GetLastError();
+        if (ec != wxSTREAM_NO_ERROR && ec != wxSTREAM_EOF)
+            throw SysError(L"wxInputStream::GetLastError (" + numberTo<std::wstring>(httpStream_->GetLastError()) + L")");
+
+        const size_t bytesRead = httpStream_->LastRead();
+        //"if there are not enough bytes in the stream right now, LastRead() value will be
+        // less than size but greater than 0. If it is 0, it means that EOF has been reached."
+        assert(bytesRead > 0 || ec == wxSTREAM_EOF);
+#endif
+        if (bytesRead > bytesToRead) //better safe than sorry
             throw SysError(L"InternetReadFile: buffer overflow.");
 
-        if (bytesRead < blockSize)
-            buffer.resize(buffer.size() - (blockSize - bytesRead)); //caveat: unsigned arithmetics
+        return bytesRead; //"zero indicates end of file"
+    }
 
-        if (bytesRead == 0)
-            return buffer;
+private:
+    Impl           (const Impl&) = delete;
+    Impl& operator=(const Impl&) = delete;
+
+    void cleanup()
+    {
+#ifdef ZEN_WIN
+        if (hRequest_ ) ::InternetCloseHandle(hRequest_);
+        if (hSession_ ) ::InternetCloseHandle(hSession_);
+        if (hInternet_) ::InternetCloseHandle(hInternet_);
+#endif
     }
 
+#ifdef ZEN_WIN
+    HINTERNET hInternet_ = nullptr;
+    HINTERNET hSession_  = nullptr;
+    HINTERNET hRequest_  = nullptr;
 #else
-    assert(std::this_thread::get_id() == mainThreadId);
-    assert(wxApp::IsMainLoopRunning());
+    wxHTTP webAccess_;
+    std::unique_ptr<wxInputStream> httpStream_; //must be deleted BEFORE webAccess is closed
+#endif
+};
 
-    wxHTTP webAccess;
-    webAccess.SetHeader(L"User-Agent", userAgent);
-    webAccess.SetTimeout(10 /*[s]*/); //default: 10 minutes: WTF are these wxWidgets people thinking???
 
-    if (!webAccess.Connect(server)) //will *not* fail for non-reachable url here!
-        throw SysError(L"wxHTTP::Connect");
+HttpInputStream::HttpInputStream(std::unique_ptr<Impl>&& pimpl) : pimpl_(std::move(pimpl)) {}
 
-    if (postParams)
-        if (!webAccess.SetPostText(L"application/x-www-form-urlencoded", utfCvrtTo<wxString>(*postParams)))
-            throw SysError(L"wxHTTP::SetPostText");
+HttpInputStream::~HttpInputStream() {}
 
-    std::unique_ptr<wxInputStream> httpStream(webAccess.GetInputStream(page)); //must be deleted BEFORE webAccess is closed
-    const int sc = webAccess.GetResponse();
+size_t HttpInputStream::tryRead(void* buffer, size_t bytesToRead) { return pimpl_->tryRead(buffer, bytesToRead); } //throw SysError
+
+
+std::string HttpInputStream::readAll() //throw SysError
+{
+    std::string buffer;
+    const size_t blockSize = getBlockSize();
 
-    //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
-    if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
+    for (;;)
     {
-        if (level < 5) //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop."
-        {
-            const std::wstring newUrl(webAccess.GetHeader(L"Location"));
-            if (!newUrl.empty())
-                return sendHttpRequestImpl(newUrl, userAgent, postParams, level + 1);
-        }
-        throw SysError(L"Unresolvable redirect.");
-    }
+        buffer.resize(buffer.size() + blockSize);
 
-    if (sc != 200) //HTTP_STATUS_OK
-        throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));
+        const size_t bytesRead = pimpl_->tryRead(&*(buffer.end() - blockSize), blockSize); //throw SysError
 
-    if (!httpStream || webAccess.GetError() != wxPROTO_NOERR)
-        throw SysError(L"wxHTTP::GetError");
+        if (bytesRead < blockSize)
+            buffer.resize(buffer.size() - (blockSize - bytesRead)); //caveat: unsigned arithmetics
 
-    std::string buffer;
-    int newValue = 0;
-    while ((newValue = httpStream->GetC()) != wxEOF)
-        buffer.push_back(static_cast<char>(newValue));
-    return buffer;
-#endif
+        if (bytesRead == 0)
+            return buffer;
+    }
+}
+
+
+namespace
+{
+std::unique_ptr<HttpInputStream::Impl> sendHttpRequestImpl(const std::wstring& url, const std::wstring& userAgent, //throw SysError
+                                                           const std::string* postParams) //issue POST if bound, GET otherwise
+{
+    std::wstring urlRed =  url;
+    //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop."
+    for (int redirects = 0; redirects < 6; ++redirects)
+        try
+        {
+            return std::make_unique<HttpInputStream::Impl>(urlRed, userAgent, postParams); //throw SysError, UrlRedirectError
+        }
+        catch (const UrlRedirectError& e) { urlRed = e.newUrl; }
+    throw SysError(L"Too many redirects.");
 }
 
 
@@ -228,31 +301,72 @@ std::string urlencode(const std::string& str)
             out += c;
         else
         {
-            const char hexDigits[] = "0123456789ABCDEF";
+            const std::pair<char, char> hex = hexify(c);
+
             out += '%';
-            out += hexDigits[static_cast<unsigned char>(c) / 16];
-            out += hexDigits[static_cast<unsigned char>(c) % 16];
+            out += hex.first;
+            out += hex.second;
+        }
+    return out;
+}
+
+
+std::string urldecode(const std::string& str)
+{
+    std::string out;
+    for (size_t i = 0; i < str.size(); ++i)
+    {
+        const char c = str[i];
+        if (c == '+')
+            out += ' ';
+        else if (c == '%' && str.size() - i >= 3 &&
+                 isHexDigit(str[i + 1]) &&
+                 isHexDigit(str[i + 2]))
+        {
+            out += unhexify(str[i + 1], str[i + 2]);
+            i += 2;
         }
+        else
+            out += c;
+    }
     return out;
 }
 }
 
 
-std::string zen::sendHttpPost(const std::wstring& url, const std::wstring& userAgent, const std::vector<std::pair<std::string, std::string>>& postParams) //throw SysError
+std::string zen::xWwwFormUrlEncode(const std::vector<std::pair<std::string, std::string>>& paramPairs)
 {
-    //convert post parameters into "application/x-www-form-urlencoded"
-    std::string flatParams;
-    for (const auto& pair : postParams)
-        flatParams += urlencode(pair.first) + '=' + urlencode(pair.second) + '&';
+    std::string output;
+    for (const auto& pair : paramPairs)
+        output += urlencode(pair.first) + '=' + urlencode(pair.second) + '&';
     //encode both key and value: https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
-    if (!flatParams.empty())
-        flatParams.pop_back();
+    if (!output.empty())
+        output.pop_back();
+    return output;
+}
 
-    return sendHttpRequestImpl(url, userAgent, &flatParams); //throw SysError
+
+std::vector<std::pair<std::string, std::string>> zen::xWwwFormUrlDecode(const std::string& str)
+{
+    std::vector<std::pair<std::string, std::string>> output;
+
+    for (const std::string& nvPair : split(str, '&'))
+        if (!nvPair.empty())
+            output.emplace_back(urldecode(beforeFirst(nvPair, '=', IF_MISSING_RETURN_ALL)),
+                                urldecode(afterFirst (nvPair, '=', IF_MISSING_RETURN_NONE)));
+    return output;
+}
+
+
+HttpInputStream zen::sendHttpPost(const std::wstring& url, const std::wstring& userAgent,
+                                  const std::vector<std::pair<std::string, std::string>>& postParams) //throw SysError
+{
+    const std::string encodedParams = xWwwFormUrlEncode(postParams);
+    return sendHttpRequestImpl(url, userAgent, &encodedParams); //throw SysError
 }
 
 
-std::string zen::sendHttpGet(const std::wstring& url, const std::wstring& userAgent) //throw SysError
+HttpInputStream zen::sendHttpGet(const std::wstring& url, const std::wstring& userAgent) //throw SysError
 {
     return sendHttpRequestImpl(url, userAgent, nullptr); //throw SysError
 }