path: root/wx+/http.cpp

// *****************************************************************************
// * This file is part of the FreeFileSync project. It is distributed under    *
// * GNU General Public License: http://www.gnu.org/licenses/gpl-3.0           *
// * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved *
// *****************************************************************************

#include "http.h"
#ifdef ZEN_WIN
    #include <zen/win.h> //tame WinINet.h include
    #include <WinINet.h>
#endif

#if defined ZEN_LINUX || defined ZEN_MAC
    #include <wx/app.h>
    #include <zen/thread.h> //std::thread::id
    #include <wx/protocol/http.h>
#endif

using namespace zen;


namespace
{
#ifdef ZEN_WIN
    #if defined NDEBUG && defined __WXWINDOWS__
        #error don not use wxWidgets for this component!
    #endif
#else
    #ifndef NDEBUG
        const std::thread::id mainThreadId = std::this_thread::get_id();
    #endif
#endif

struct UrlRedirectError
{
    UrlRedirectError(const std::wstring& url) : newUrl(url) {}
    std::wstring newUrl;
};
}


class HttpInputStream::Impl
{
public:
    Impl(const std::wstring& url, const std::wstring& userAgent, //throw SysError, UrlRedirectError
         const std::string* postParams) //issue POST if bound, GET otherwise
    {
        ZEN_ON_SCOPE_FAIL( cleanup(); /*destructor call would lead to member double clean-up!!!*/ );

        assert(!startsWith(makeUpperCopy(url), L"HTTPS:")); //not supported by wxHTTP!
        const std::wstring urlFmt = startsWith(makeUpperCopy(url), L"HTTP://") ||
                                    startsWith(makeUpperCopy(url), L"HTTPS://") ? afterFirst(url, L"://", IF_MISSING_RETURN_NONE) : url;
        const std::wstring server =       beforeFirst(urlFmt, L'/', IF_MISSING_RETURN_ALL);
        const std::wstring page   = L'/' + afterFirst(urlFmt, L'/', IF_MISSING_RETURN_NONE);

#ifdef ZEN_WIN
        //WinInet: 1. uses IE proxy settings! :) 2. follows HTTP redirects by default 3. swallows HTTPS if needed
        hInternet_ = ::InternetOpen(userAgent.c_str(),            //_In_  LPCTSTR lpszAgent,
                                    INTERNET_OPEN_TYPE_PRECONFIG, //_In_  DWORD dwAccessType,
                                    nullptr, //_In_  LPCTSTR lpszProxyName,
                                    nullptr, //_In_  LPCTSTR lpszProxyBypass,
                                    0);      //_In_  DWORD dwFlags
        if (!hInternet_)
            THROW_LAST_SYS_ERROR(L"InternetOpen");

        hSession_ = ::InternetConnect(hInternet_,     //_In_ HINTERNET     hInternet,
                                      server.c_str(), //_In_ LPCTSTR       lpszServerName,
                                      INTERNET_DEFAULT_HTTP_PORT, //_In_ INTERNET_PORT nServerPort,
                                      nullptr,        //_In_ LPCTSTR       lpszUsername,
                                      nullptr,        //_In_ LPCTSTR       lpszPassword,
                                      INTERNET_SERVICE_HTTP, //_In_ DWORD         dwService,
                                      0,              //_In_ DWORD         dwFlags,
                                      0);             //_In_ DWORD_PTR     dwContext
        if (!hSession_)
            THROW_LAST_SYS_ERROR(L"InternetConnect");

        const wchar_t* acceptTypes[] = { L"*/*", nullptr };
        DWORD requestFlags =
            //INTERNET_FLAG_KEEP_CONNECTION |
            //      the combination 1. INTERNET_FLAG_KEEP_CONNECTION (= adds "Connection: Keep-Alive" but NOT "Keep-Alive: timeout" to the header)
            //                      2. *no* "Keep-Alive: timeout" header entry 3. call from within VM and 4. *no* Fiddler running 5. HTTP POST
            //      leads to Godaddy blocking the IP: http://www.freefilesync.org/forum/viewtopic.php?t=3855
            //      => it seems a broken keep alive header is the trigger: But why is it then working outside the VM or when Fiddler is running??? Why not a problem for HTTP GET?
            //          note: HTTP/1.1 has keep-alive semantics by default, so this flag is probably useless anyway
            INTERNET_FLAG_NO_UI;

        if (postParams)
        {
            requestFlags |= INTERNET_FLAG_NO_AUTO_REDIRECT; //POST would be re-issued as GET during auto-redirect => handle ourselves!
        }
        else //HTTP GET
        {
            requestFlags |= INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP;
            requestFlags |= INTERNET_FLAG_RELOAD; //not relevant for POST (= never cached)
        }

        hRequest_ = ::HttpOpenRequest(hSession_,    //_In_ HINTERNET hConnect,
                                      postParams ? L"POST" : L"GET", //_In_ LPCTSTR   lpszVerb,
                                      page.c_str(), //_In_ LPCTSTR   lpszObjectName,
                                      nullptr,      //_In_ LPCTSTR   lpszVersion,
                                      nullptr,      //_In_ LPCTSTR   lpszReferer,
                                      acceptTypes,  //_In_ LPCTSTR   *lplpszAcceptTypes,
                                      requestFlags, //_In_ DWORD     dwFlags,
                                      0); //_In_ DWORD_PTR dwContext
        if (!hRequest_)
            THROW_LAST_SYS_ERROR(L"HttpOpenRequest");

        const std::wstring headers = postParams ? L"Content-type: application/x-www-form-urlencoded" : L"";

        assert(std::all_of(headers.begin(), headers.end(), [](wchar_t c) { return makeUnsigned(c) < 128; }));
        //HttpSendRequest has finicky behavior for non-ASCII headers: https://msdn.microsoft.com/en-us/library/windows/desktop/aa384247

        std::string postParamsBuf = postParams ? *postParams : "";

        if (!::HttpSendRequest(hRequest_,             //_In_ HINTERNET hRequest,
                               headers.c_str(),       //_In_ LPCTSTR   lpszHeaders,
                               static_cast<DWORD>(headers.size()),                  //_In_ DWORD     dwHeadersLength,
                               postParamsBuf.empty() ? nullptr : &postParamsBuf[0], //_In_ LPVOID    lpOptional,
                               static_cast<DWORD>(postParamsBuf.size())))           //_In_ DWORD     dwOptionalLength
            THROW_LAST_SYS_ERROR(L"HttpSendRequest");

        DWORD sc = 0;
        {
            DWORD bufLen = sizeof(sc);
            if (!::HttpQueryInfo(hRequest_, //_In_     HINTERNET hRequest,
                                 HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, //_In_     DWORD dwInfoLevel,
                                 &sc,       //_Inout_  LPVOID lpvBuffer,
                                 &bufLen,   //_Inout_  LPDWORD lpdwBufferLength,
                                 nullptr))  //_Inout_  LPDWORD lpdwIndex
                THROW_LAST_SYS_ERROR(L"HttpQueryInfo: HTTP_QUERY_STATUS_CODE");
        }

        //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
        if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
        {
            DWORD bufLen = 10000;
            std::wstring location(bufLen, L'\0');
            if (!::HttpQueryInfo(hRequest_, HTTP_QUERY_LOCATION, &*location.begin(), &bufLen, nullptr))
                THROW_LAST_SYS_ERROR(L"HttpQueryInfo: HTTP_QUERY_LOCATION");
            if (bufLen >= location.size()) //HttpQueryInfo expected to write terminating zero
                throw SysError(L"HttpQueryInfo: HTTP_QUERY_LOCATION, buffer overflow");
            location.resize(bufLen);

            if (location.empty())
                throw SysError(L"Unresolvable redirect. Empty target Location.");

            throw UrlRedirectError(location);
        }

        if (sc != HTTP_STATUS_OK) //200
            throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));
        //e.g. 404 - HTTP_STATUS_NOT_FOUND

#else
        assert(std::this_thread::get_id() == mainThreadId);
        assert(wxApp::IsMainLoopRunning());

        webAccess_.SetHeader(L"User-Agent", userAgent);
        webAccess_.SetTimeout(10 /*[s]*/); //default: 10 minutes: WTF are these wxWidgets people thinking???

        if (!webAccess_.Connect(server)) //will *not* fail for non-reachable url here!
            throw SysError(L"wxHTTP::Connect");

        if (postParams)
            if (!webAccess_.SetPostText(L"application/x-www-form-urlencoded", utfCvrtTo<wxString>(*postParams)))
                throw SysError(L"wxHTTP::SetPostText");

        httpStream_.reset(webAccess_.GetInputStream(page)); //pass ownership
        const int sc = webAccess_.GetResponse();

        //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
        if (sc / 100 == 3) //e.g. 301, 302, 303, 307... we're not too greedy since we check location, too!
        {
            const std::wstring newUrl(webAccess_.GetHeader(L"Location"));
            if (newUrl.empty())
                throw SysError(L"Unresolvable redirect. Empty target Location.");

            throw UrlRedirectError(newUrl);
        }

        if (sc != 200) //HTTP_STATUS_OK
            throw SysError(replaceCpy<std::wstring>(L"HTTP status code %x.", L"%x", numberTo<std::wstring>(sc)));

        if (!httpStream_ || webAccess_.GetError() != wxPROTO_NOERR)
            throw SysError(L"wxHTTP::GetError (" + numberTo<std::wstring>(webAccess_.GetError()) + L")");
#endif
    }

    ~Impl() { cleanup(); }

    size_t tryRead(void* buffer, size_t bytesToRead) //throw SysError; may return short, only 0 means EOF!
    {
        if (bytesToRead == 0) //"read() with a count of 0 returns zero" => indistinguishable from end of file! => check!
            throw std::logic_error("Contract violation! " + std::string(__FILE__) + ":" + numberTo<std::string>(__LINE__));

#ifdef ZEN_WIN
        //"HttpQueryInfo() + HTTP_QUERY_CONTENT_LENGTH" not supported by all http servers...
        DWORD bytesRead = 0;
        if (!::InternetReadFile(hRequest_,   //_In_   HINTERNET hFile,
                                buffer,      //_Out_  LPVOID lpBuffer,
                                static_cast<DWORD>(bytesToRead), //_In_   DWORD dwNumberOfBytesToRead,
                                &bytesRead)) //_Out_  LPDWORD lpdwNumberOfBytesRead
            THROW_LAST_SYS_ERROR(L"InternetReadFile");
#else
        httpStream_->Read(buffer, bytesToRead);

        const wxStreamError ec = httpStream_->GetLastError();
        if (ec != wxSTREAM_NO_ERROR && ec != wxSTREAM_EOF)
            throw SysError(L"wxInputStream::GetLastError (" + numberTo<std::wstring>(httpStream_->GetLastError()) + L")");

        const size_t bytesRead = httpStream_->LastRead();
        //"if there are not enough bytes in the stream right now, LastRead() value will be
        // less than size but greater than 0. If it is 0, it means that EOF has been reached."
        assert(bytesRead > 0 || ec == wxSTREAM_EOF);
#endif
        if (bytesRead > bytesToRead) //better safe than sorry
            throw SysError(L"InternetReadFile: buffer overflow.");

        return bytesRead; //"zero indicates end of file"
    }

private:
    Impl           (const Impl&) = delete;
    Impl& operator=(const Impl&) = delete;

    void cleanup()
    {
#ifdef ZEN_WIN
        if (hRequest_ ) ::InternetCloseHandle(hRequest_);
        if (hSession_ ) ::InternetCloseHandle(hSession_);
        if (hInternet_) ::InternetCloseHandle(hInternet_);
#endif
    }

#ifdef ZEN_WIN
    HINTERNET hInternet_ = nullptr;
    HINTERNET hSession_  = nullptr;
    HINTERNET hRequest_  = nullptr;
#else
    wxHTTP webAccess_;
    std::unique_ptr<wxInputStream> httpStream_; //must be deleted BEFORE webAccess is closed
#endif
};


HttpInputStream::HttpInputStream(std::unique_ptr<Impl>&& pimpl) : pimpl_(std::move(pimpl)) {}

HttpInputStream::~HttpInputStream() {}

size_t HttpInputStream::tryRead(void* buffer, size_t bytesToRead) { return pimpl_->tryRead(buffer, bytesToRead); } //throw SysError


std::string HttpInputStream::readAll() //throw SysError
{
    std::string buffer;
    const size_t blockSize = getBlockSize();

    for (;;)
    {
        buffer.resize(buffer.size() + blockSize);

        const size_t bytesRead = pimpl_->tryRead(&*(buffer.end() - blockSize), blockSize); //throw SysError

        if (bytesRead < blockSize)
            buffer.resize(buffer.size() - (blockSize - bytesRead)); //caveat: unsigned arithmetics

        if (bytesRead == 0)
            return buffer;
    }
}


namespace
{
std::unique_ptr<HttpInputStream::Impl> sendHttpRequestImpl(const std::wstring& url, const std::wstring& userAgent, //throw SysError
                                                           const std::string* postParams) //issue POST if bound, GET otherwise
{
    std::wstring urlRed =  url;
    //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop."
    for (int redirects = 0; redirects < 6; ++redirects)
        try
        {
            return std::make_unique<HttpInputStream::Impl>(urlRed, userAgent, postParams); //throw SysError, UrlRedirectError
        }
        catch (const UrlRedirectError& e) { urlRed = e.newUrl; }
    throw SysError(L"Too many redirects.");
}


//encode into "application/x-www-form-urlencoded"
std::string urlencode(const std::string& str)
{
    std::string out;
    for (const char c : str) //follow PHP spec: https://github.com/php/php-src/blob/master/ext/standard/url.c#L500
        if (c == ' ')
            out += '+';
        else if (('0' <= c && c <= '9') ||
                 ('A' <= c && c <= 'Z') ||
                 ('a' <= c && c <= 'z') ||
                 c == '-' || c == '.' || c == '_') //note: "~" is encoded by PHP!
            out += c;
        else
        {
            const std::pair<char, char> hex = hexify(c);

            out += '%';
            out += hex.first;
            out += hex.second;
        }
    return out;
}


std::string urldecode(const std::string& str)
{
    std::string out;
    for (size_t i = 0; i < str.size(); ++i)
    {
        const char c = str[i];
        if (c == '+')
            out += ' ';
        else if (c == '%' && str.size() - i >= 3 &&
                 isHexDigit(str[i + 1]) &&
                 isHexDigit(str[i + 2]))
        {
            out += unhexify(str[i + 1], str[i + 2]);
            i += 2;
        }
        else
            out += c;
    }
    return out;
}
}


std::string zen::xWwwFormUrlEncode(const std::vector<std::pair<std::string, std::string>>& paramPairs)
{
    std::string output;
    for (const auto& pair : paramPairs)
        output += urlencode(pair.first) + '=' + urlencode(pair.second) + '&';
    //encode both key and value: https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
    if (!output.empty())
        output.pop_back();
    return output;
}


std::vector<std::pair<std::string, std::string>> zen::xWwwFormUrlDecode(const std::string& str)
{
    std::vector<std::pair<std::string, std::string>> output;

    for (const std::string& nvPair : split(str, '&'))
        if (!nvPair.empty())
            output.emplace_back(urldecode(beforeFirst(nvPair, '=', IF_MISSING_RETURN_ALL)),
                                urldecode(afterFirst (nvPair, '=', IF_MISSING_RETURN_NONE)));
    return output;
}


HttpInputStream zen::sendHttpPost(const std::wstring& url, const std::wstring& userAgent,
                                  const std::vector<std::pair<std::string, std::string>>& postParams) //throw SysError
{
    const std::string encodedParams = xWwwFormUrlEncode(postParams);
    return sendHttpRequestImpl(url, userAgent, &encodedParams); //throw SysError
}


HttpInputStream zen::sendHttpGet(const std::wstring& url, const std::wstring& userAgent) //throw SysError
{
    return sendHttpRequestImpl(url, userAgent, nullptr); //throw SysError
}


bool zen::internetIsAlive() //noexcept
{
#ifdef ZEN_WIN
    //::InternetAttemptConnect(0) -> not working as expected: succeeds even when there is no internet connection!

    HINTERNET hInternet = ::InternetOpen(L"FreeFileSync", //_In_  LPCTSTR lpszAgent,
                                         INTERNET_OPEN_TYPE_PRECONFIG, //_In_  DWORD dwAccessType,
                                         nullptr, //_In_  LPCTSTR lpszProxyName,
                                         nullptr, //_In_  LPCTSTR lpszProxyBypass,
                                         0);      //_In_  DWORD dwFlags
    if (!hInternet)
        return false;
    ZEN_ON_SCOPE_EXIT(::InternetCloseHandle(hInternet));

    //InternetOpenUrl is shortcut for HTTP:GET with InternetConnect + HttpOpenRequest + HttpSendRequest:
    HINTERNET hRequest = ::InternetOpenUrl(hInternet,   //_In_  HINTERNET hInternet,
                                           L"http://www.google.com/", //_In_  LPCTSTR lpszUrl,
                                           nullptr,     //_In_  LPCTSTR lpszHeaders,
                                           0,           //_In_  DWORD dwHeadersLength,
                                           INTERNET_FLAG_KEEP_CONNECTION |
                                           INTERNET_FLAG_NO_UI  |
                                           INTERNET_FLAG_RELOAD |
                                           INTERNET_FLAG_NO_AUTO_REDIRECT, //_In_  DWORD dwFlags,
                                           0);          //_In_  DWORD_PTR dwContext
    //fails with ERROR_INTERNET_NAME_NOT_RESOLVED if server not found => the server-relative part is checked by HTTP_QUERY_STATUS_CODE!!!
    if (!hRequest)
        return false;
    ZEN_ON_SCOPE_EXIT(::InternetCloseHandle(hRequest));

    DWORD sc = 0;
    {
        DWORD bufLen = sizeof(sc);
        if (!::HttpQueryInfo(hRequest, //_In_     HINTERNET hRequest,
                             HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, //_In_     DWORD dwInfoLevel,
                             &sc,      //_Inout_  LPVOID lpvBuffer,
                             &bufLen,  //_Inout_  LPDWORD lpdwBufferLength,
                             nullptr)) //_Inout_  LPDWORD lpdwIndex
            return false;
    }

#else
    assert(std::this_thread::get_id() == mainThreadId);

    const wxString server = L"www.google.com";
    const wxString page   = L"/";

    wxHTTP webAccess;
    webAccess.SetTimeout(10 /*[s]*/); //default: 10 minutes: WTF are these wxWidgets people thinking???

    if (!webAccess.Connect(server)) //will *not* fail for non-reachable url here!
        return false;

    std::unique_ptr<wxInputStream> httpStream(webAccess.GetInputStream(page)); //call before checking wxHTTP::GetResponse()
    const int sc = webAccess.GetResponse();
#endif
    //attention: http://www.google.com/ might redirect to "https" => don't follow, just return "true"!!!
    return sc / 100 == 2 || //e.g. 200
           sc / 100 == 3;   //e.g. 301, 302, 303, 307... when in doubt, consider internet alive!
}