add upstream 10.14

4 files changed, 65 insertions, 38 deletions

diff --git a/zen/http.cpp b/zen/http.cpp
index f8538c93..43c9dcbf 100644
--- a/zen/http.cpp
+++ b/zen/http.cpp
@@ -7,6 +7,7 @@
 #include "http.h"
 
     #include "socket.h"
+    #include "open_ssl.h"
 
 using namespace zen;
 
@@ -14,11 +15,15 @@ using namespace zen;
 class HttpInputStream::Impl
 {
 public:
-    Impl(const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO, //throw SysError
-         const std::vector<std::pair<std::string, std::string>>* postParams) : //issue POST if bound, GET otherwise
+    Impl(const Zstring& url,
+         const std::vector<std::pair<std::string, std::string>>* postParams, //issue POST if bound, GET otherwise
+         bool disableGetCache /*not relevant for POST (= never cached)*/,
+         const Zstring& userAgent,
+         const Zstring* caCertFilePath /*optional: enable certificate validation*/,
+         const IOCallback& notifyUnbufferedIO) : //throw SysError
         notifyUnbufferedIO_(notifyUnbufferedIO)
     {
-        ZEN_ON_SCOPE_FAIL( cleanup(); /*destructor call would lead to member double clean-up!!!*/ );
+        ZEN_ON_SCOPE_FAIL(cleanup(); /*destructor call would lead to member double clean-up!!!*/);
 
         const Zstring urlFmt =              afterFirst(url, Zstr("://"), IF_MISSING_RETURN_NONE);
         const Zstring server =            beforeFirst(urlFmt, Zstr('/'), IF_MISSING_RETURN_ALL);
@@ -33,12 +38,15 @@ public:
             throw SysError(L"URL uses unexpected protocol.");
         }();
 
-        assert(!useTls); //not supported by our plain socket!
-        (void)useTls;
-
-        socket_ = std::make_unique<Socket>(server, Zstr("http")); //throw SysError
-        //HTTP default port: 80, see %WINDIR%\system32\drivers\etc\services
+        if (useTls) //HTTP default port: 443, see %WINDIR%\system32\drivers\etc\services
+        {
+            socket_ = std::make_unique<Socket>(server, Zstr("https")); //throw SysError
+            tlsCtx_ = std::make_unique<TlsContext>(socket_->get(), server, caCertFilePath); //throw SysError
+        }
+        else //HTTP default port: 80, see %WINDIR%\system32\drivers\etc\services
+            socket_ = std::make_unique<Socket>(server, Zstr("http")); //throw SysError
 
+		//we don't support "chunked transfer encoding" => HTTP 1.0
         std::map<std::string, std::string, LessAsciiNoCase> headers;
         headers["Host"      ] = utfTo<std::string>(server); //only required for HTTP/1.1
         headers["User-Agent"] = utfTo<std::string>(userAgent);
@@ -46,12 +54,11 @@ public:
 
         const std::string postBuf = postParams ? xWwwFormUrlEncode(*postParams) : "";
 
-        if (!postParams) //HTTP GET
+        if (!postParams /*HTTP GET*/ && disableGetCache)
             headers["Pragma"] = "no-cache"; //HTTP 1.0 only! superseeded by "Cache-Control"
-        //consider internetIsAlive() test; not relevant for POST (= never cached)
         else //HTTP POST
         {
-            headers["Content-type"] = "application/x-www-form-urlencoded";
+            headers["Content-Type"] = "application/x-www-form-urlencoded";
             headers["Content-Length"] = numberTo<std::string>(postBuf.size());
         }
 
@@ -64,9 +71,13 @@ public:
 
         //send request
         for (size_t bytesToSend = msg.size(); bytesToSend > 0;)
-            bytesToSend -= tryWriteSocket(socket_->get(), &*(msg.end() - bytesToSend), bytesToSend); //throw SysError
+            bytesToSend -= tlsCtx_ ?
+                           tlsCtx_->tryWrite(             &*(msg.end() - bytesToSend), bytesToSend) : //throw SysError
+                           tryWriteSocket(socket_->get(), &*(msg.end() - bytesToSend), bytesToSend);  //throw SysError
 
-        shutdownSocketSend(socket_->get()); //throw SysError
+        //shutdownSocketSend(socket_->get()); //throw SysError
+        //NO! Sending TCP FIN before receiving response (aka "TCP Half Closed") is not always supported! e.g. Cloudflare server will immediately end connection: recv() returns 0.
+        //"clients SHOULD NOT half-close their TCP connections": https://github.com/httpwg/http-core/issues/22
 
         //receive response:
         std::string headBuf;
@@ -164,7 +175,9 @@ private:
                 return 0;
             bytesToRead = static_cast<size_t>(std::min(static_cast<int64_t>(bytesToRead), contentRemaining_)); //[!] contentRemaining_ > 4 GB possible!
         }
-        const size_t bytesReceived = tryReadSocket(socket_->get(), buffer, bytesToRead); //throw SysError; may return short, only 0 means EOF!
+        const size_t bytesReceived = tlsCtx_ ?
+                                     tlsCtx_->tryRead(                buffer, bytesToRead) : //throw SysError; may return short, only 0 means EOF!
+                                     tryReadSocket   (socket_->get(), buffer, bytesToRead);  //
         if (contentRemaining_ >= 0)
             contentRemaining_ -= bytesReceived;
 
@@ -174,14 +187,15 @@ private:
         return bytesReceived; //"zero indicates end of file"
     }
 
-    Impl           (const Impl&) = delete;
-    Impl& operator=(const Impl&) = delete;
-
     void cleanup()
     {
     }
 
-    std::unique_ptr<Socket> socket_; //*bound* after constructor has run
+    Impl           (const Impl&) = delete;
+    Impl& operator=(const Impl&) = delete;
+
+    std::unique_ptr<Socket> socket_;     //*bound* after constructor has run
+    std::unique_ptr<TlsContext> tlsCtx_; //optional: support HTTPS
     int statusCode_ = 0;
     std::map<std::string, std::string, LessAsciiNoCase> responseHeaders_;
 
@@ -208,14 +222,17 @@ std::string HttpInputStream::readAll() { return bufferedLoad<std::string>(*pimpl
 
 namespace
 {
-std::unique_ptr<HttpInputStream::Impl> sendHttpRequestImpl(const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO, //throw SysError
-                                                           const std::vector<std::pair<std::string, std::string>>* postParams) //issue POST if bound, GET otherwise
+std::unique_ptr<HttpInputStream::Impl> sendHttpRequestImpl(const Zstring& url,
+                                                           const std::vector<std::pair<std::string, std::string>>* postParams /*issue POST if bound, GET otherwise*/,
+                                                           const Zstring& userAgent,
+                                                           const Zstring* caCertFilePath /*optional: enable certificate validation*/,
+                                                           const IOCallback& notifyUnbufferedIO) //throw SysError
 {
     Zstring urlRed = url;
     //"A user agent should not automatically redirect a request more than five times, since such redirections usually indicate an infinite loop."
     for (int redirects = 0; redirects < 6; ++redirects)
     {
-        auto response = std::make_unique<HttpInputStream::Impl>(urlRed, userAgent, notifyUnbufferedIO, postParams); //throw SysError
+        auto response = std::make_unique<HttpInputStream::Impl>(urlRed, postParams, false /*disableGetCache*/, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError
 
         //http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_Redirection
         const int statusCode = response->getStatusCode();
@@ -310,16 +327,16 @@ std::vector<std::pair<std::string, std::string>> zen::xWwwFormUrlDecode(const st
 }
 
 
-HttpInputStream zen::sendHttpPost(const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO,
-                                  const std::vector<std::pair<std::string, std::string>>& postParams) //throw SysError
+HttpInputStream zen::sendHttpPost(const Zstring& url, const std::vector<std::pair<std::string, std::string>>& postParams,
+	const Zstring& userAgent, const Zstring* caCertFilePath, const IOCallback& notifyUnbufferedIO) //throw SysError
 {
-    return sendHttpRequestImpl(url, userAgent, notifyUnbufferedIO, &postParams); //throw SysError
+    return sendHttpRequestImpl(url, &postParams, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError
 }
 
 
-HttpInputStream zen::sendHttpGet(const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO) //throw SysError
+HttpInputStream zen::sendHttpGet(const Zstring& url, const Zstring& userAgent, const Zstring* caCertFilePath, const IOCallback& notifyUnbufferedIO) //throw SysError
 {
-    return sendHttpRequestImpl(url, userAgent, notifyUnbufferedIO, nullptr); //throw SysError
+    return sendHttpRequestImpl(url, nullptr /*postParams*/, userAgent, caCertFilePath, notifyUnbufferedIO); //throw SysError
 }
 
 
@@ -328,9 +345,11 @@ bool zen::internetIsAlive() //noexcept
     try
     {
         auto response = std::make_unique<HttpInputStream::Impl>(Zstr("http://www.google.com/"),
+                                                                nullptr /*postParams*/,
+                                                                true /*disableGetCache*/,
                                                                 Zstr("FreeFileSync"),
-                                                                nullptr /*notifyUnbufferedIO*/,
-                                                                nullptr /*postParams*/); //throw SysError
+                                                                nullptr /*caCertFilePath*/,
+                                                                nullptr /*notifyUnbufferedIO*/); //throw SysError
         const int statusCode = response->getStatusCode();
 
         //attention: http://www.google.com/ might redirect to "https" => don't follow, just return "true"!!!
diff --git a/zen/http.h b/zen/http.h
index 5d84be2c..2fafa574 100644
--- a/zen/http.h
+++ b/zen/http.h
@@ -15,7 +15,7 @@ namespace zen
 {
 /*
     - thread-safe! (Window/Linux/macOS)
-    - HTTPS supported only for Windows
+    - Linux/macOS: init OpenSSL before use!
 */
 class HttpInputStream
 {
@@ -36,9 +36,16 @@ private:
 };
 
 
-HttpInputStream sendHttpGet (const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO /*throw X*/); //throw SysError
-HttpInputStream sendHttpPost(const Zstring& url, const Zstring& userAgent, const IOCallback& notifyUnbufferedIO /*throw X*/,  //
-                             const std::vector<std::pair<std::string, std::string>>& postParams);
+HttpInputStream sendHttpGet(const Zstring& url,
+                            const Zstring& userAgent,
+                            const Zstring* caCertFilePath /*optional: enable certificate validation*/,
+                            const IOCallback& notifyUnbufferedIO /*throw X*/); //throw SysError
+
+HttpInputStream sendHttpPost(const Zstring& url,
+                             const std::vector<std::pair<std::string, std::string>>& postParams,
+                             const Zstring& userAgent,
+                             const Zstring* caCertFilePath /*optional: enable certificate validation*/,
+                             const IOCallback& notifyUnbufferedIO /*throw X*/);
 bool internetIsAlive(); //noexcept
 
 std::string xWwwFormUrlEncode(const std::vector<std::pair<std::string, std::string>>& paramPairs);
diff --git a/zen/socket.h b/zen/socket.h
index 0b4c823d..7ca0c93f 100644
--- a/zen/socket.h
+++ b/zen/socket.h
@@ -105,7 +105,7 @@ size_t tryReadSocket(SocketType socket, void* buffer, size_t bytesToRead) //thro
         THROW_LAST_SYS_ERROR_WSA(L"recv");
 
     if (static_cast<size_t>(bytesReceived) > bytesToRead) //better safe than sorry
-        throw SysError(L"HttpInputStream::tryRead: buffer overflow.");
+        throw SysError(L"recv: buffer overflow.");
 
     return bytesReceived; //"zero indicates end of file"
 }
@@ -138,10 +138,11 @@ size_t tryWriteSocket(SocketType socket, const void* buffer, size_t bytesToWrite
 }
 
 
+//initiate termination of connection by sending TCP FIN package
 inline
 void shutdownSocketSend(SocketType socket) //throw SysError
 {
-    if (::shutdown(socket, SHUT_WR) != 0)
+    if (::shutdown(socket, SHUT_WR) != 0) 
         THROW_LAST_SYS_ERROR_WSA(L"shutdown");
 }
 
diff --git a/zen/warn_static.h b/zen/warn_static.h
index fb8fbb95..17e7cf25 100644
--- a/zen/warn_static.h
+++ b/zen/warn_static.h
@@ -8,10 +8,10 @@
 #define WARN_STATIC_H_08724567834560832745
 
 /*
-Portable Compile-Time Warning
------------------------------
-Usage:
-    warn_static("my message")
+	Portable Compile-Time Warning
+	-----------------------------
+	Usage:
+		warn_static("my message")
 */
 
 #define ZEN_STATIC_WARNING_STRINGIZE(NUM) #NUM