httpserver add gzip (continue)

hikyuu_cpp/hikyuu/utilities/arithmetic.h

@@ -12,7 +12,9 @@
 #define HIKYUU_UTILITIES_ARITHMETIC_H
 
 #include <cctype>
+#include <vector>
 #include <string>
+#include <string_view>
 #include <algorithm>
 
 namespace hku {
@@ -45,6 +47,45 @@ inline void trim(std::string& s) {
     s.erase(s.find_last_not_of(" ") + 1);
 }
 
+/**
+ * 分割字符串
+ * @param str 待封的字符串
+ * @param c 分割符
+ */
+inline std::vector<std::string_view> split(const std::string& str, char c) {
+    std::vector<std::string_view> result;
+    std::string_view view(str);
+    size_t prepos = 0;
+    size_t pos = view.find_first_of(c);
+    while (pos != std::string::npos) {
+        result.emplace_back(view.substr(prepos, pos - prepos));
+        prepos = pos + 1;
+        pos = view.find_first_of(c, prepos);
+    }
+
+    result.emplace_back(view.substr(prepos));
+    return result;
+}
+
+/**
+ * 字符串分割
+ * @param view 待分割的 string_view
+ * @param c 分割符
+ */
+inline std::vector<std::string_view> split(const std::string_view& view, char c) {
+    std::vector<std::string_view> result;
+    size_t prepos = 0;
+    size_t pos = view.find_first_of(c);
+    while (pos != std::string::npos) {
+        result.emplace_back(view.substr(prepos, pos - prepos));
+        prepos = pos + 1;
+        pos = view.find_first_of(c, prepos);
+    }
+
+    result.emplace_back(view.substr(prepos));
+    return result;
+}
+
 /** @} */
 } /* namespace hku */
 

hikyuu_cpp/hikyuu_server/gzip/compress.hpp

@@ -0,0 +1,102 @@
+#include <gzip/config.hpp>
+
+// zlib
+#include <zlib.h>
+
+// std
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+namespace gzip {
+
+class Compressor {
+    std::size_t max_;
+    int level_;
+
+public:
+    Compressor(int level = Z_DEFAULT_COMPRESSION,
+               std::size_t max_bytes =
+                 2000000000)  // by default refuse operation if uncompressed data is > 2GB
+    : max_(max_bytes), level_(level) {}
+
+    template <typename InputType>
+    void compress(InputType& output, const char* data, std::size_t size) const {
+#ifdef DEBUG
+        // Verify if size input will fit into unsigned int, type used for zlib's avail_in
+        if (size > std::numeric_limits<unsigned int>::max()) {
+            throw std::runtime_error("size arg is too large to fit into unsigned int type");
+        }
+#endif
+        if (size > max_) {
+            throw std::runtime_error("size may use more memory than intended when decompressing");
+        }
+
+        z_stream deflate_s;
+        deflate_s.zalloc = Z_NULL;
+        deflate_s.zfree = Z_NULL;
+        deflate_s.opaque = Z_NULL;
+        deflate_s.avail_in = 0;
+        deflate_s.next_in = Z_NULL;
+
+        // The windowBits parameter is the base two logarithm of the window size (the size of the
+        // history buffer). It should be in the range 8..15 for this version of the library. Larger
+        // values of this parameter result in better compression at the expense of memory usage.
+        // This range of values also changes the decoding type:
+        //  -8 to -15 for raw deflate
+        //  8 to 15 for zlib
+        // (8 to 15) + 16 for gzip
+        // (8 to 15) + 32 to automatically detect gzip/zlib header (decompression/inflate only)
+        constexpr int window_bits = 15 + 16;  // gzip with windowbits of 15
+
+        constexpr int mem_level = 8;
+        // The memory requirements for deflate are (in bytes):
+        // (1 << (window_bits+2)) +  (1 << (mem_level+9))
+        // with a default value of 8 for mem_level and our window_bits of 15
+        // this is 128Kb
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#endif
+        if (deflateInit2(&deflate_s, level_, Z_DEFLATED, window_bits, mem_level,
+                         Z_DEFAULT_STRATEGY) != Z_OK) {
+            throw std::runtime_error("deflate init failed");
+        }
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
+        deflate_s.next_in = reinterpret_cast<z_const Bytef*>(data);
+        deflate_s.avail_in = static_cast<unsigned int>(size);
+
+        std::size_t size_compressed = 0;
+        do {
+            size_t increase = size / 2 + 1024;
+            if (output.size() < (size_compressed + increase)) {
+                output.resize(size_compressed + increase);
+            }
+            // There is no way we see that "increase" would not fit in an unsigned int,
+            // hence we use static cast here to avoid -Wshorten-64-to-32 error
+            deflate_s.avail_out = static_cast<unsigned int>(increase);
+            deflate_s.next_out = reinterpret_cast<Bytef*>((&output[0] + size_compressed));
+            // From http://www.zlib.net/zlib_how.html
+            // "deflate() has a return value that can indicate errors, yet we do not check it here.
+            // Why not? Well, it turns out that deflate() can do no wrong here."
+            // Basically only possible error is from deflateInit not working properly
+            deflate(&deflate_s, Z_FINISH);
+            size_compressed += (increase - deflate_s.avail_out);
+        } while (deflate_s.avail_out == 0);
+
+        deflateEnd(&deflate_s);
+        output.resize(size_compressed);
+    }
+};
+
+inline std::string compress(const char* data, std::size_t size, int level = Z_DEFAULT_COMPRESSION) {
+    Compressor comp(level);
+    std::string output;
+    comp.compress(output, data, size);
+    return output;
+}
+
+}  // namespace gzip

hikyuu_cpp/hikyuu_server/gzip/config.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+#ifndef ZLIB_CONST
+#define ZLIB_CONST
+#endif

hikyuu_cpp/hikyuu_server/gzip/decompress.hpp

@@ -0,0 +1,100 @@
+#include <gzip/config.hpp>
+
+// zlib
+#include <zlib.h>
+
+// std
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+namespace gzip {
+
+class Decompressor {
+    std::size_t max_;
+
+public:
+    Decompressor(std::size_t max_bytes =
+                   1000000000)  // by default refuse operation if compressed data is > 1GB
+    : max_(max_bytes) {}
+
+    template <typename OutputType>
+    void decompress(OutputType& output, const char* data, std::size_t size) const {
+        z_stream inflate_s;
+
+        inflate_s.zalloc = Z_NULL;
+        inflate_s.zfree = Z_NULL;
+        inflate_s.opaque = Z_NULL;
+        inflate_s.avail_in = 0;
+        inflate_s.next_in = Z_NULL;
+
+        // The windowBits parameter is the base two logarithm of the window size (the size of the
+        // history buffer). It should be in the range 8..15 for this version of the library. Larger
+        // values of this parameter result in better compression at the expense of memory usage.
+        // This range of values also changes the decoding type:
+        //  -8 to -15 for raw deflate
+        //  8 to 15 for zlib
+        // (8 to 15) + 16 for gzip
+        // (8 to 15) + 32 to automatically detect gzip/zlib header
+        constexpr int window_bits = 15 + 32;  // auto with windowbits of 15
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#endif
+        if (inflateInit2(&inflate_s, window_bits) != Z_OK) {
+            throw std::runtime_error("inflate init failed");
+        }
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
+        inflate_s.next_in = reinterpret_cast<z_const Bytef*>(data);
+
+#ifdef DEBUG
+        // Verify if size (long type) input will fit into unsigned int, type used for zlib's
+        // avail_in
+        std::uint64_t size_64 = size * 2;
+        if (size_64 > std::numeric_limits<unsigned int>::max()) {
+            inflateEnd(&inflate_s);
+            throw std::runtime_error("size arg is too large to fit into unsigned int type x2");
+        }
+#endif
+        if (size > max_ || (size * 2) > max_) {
+            inflateEnd(&inflate_s);
+            throw std::runtime_error("size may use more memory than intended when decompressing");
+        }
+        inflate_s.avail_in = static_cast<unsigned int>(size);
+        std::size_t size_uncompressed = 0;
+        do {
+            std::size_t resize_to = size_uncompressed + 2 * size;
+            if (resize_to > max_) {
+                inflateEnd(&inflate_s);
+                throw std::runtime_error(
+                  "size of output string will use more memory then intended when decompressing");
+            }
+            output.resize(resize_to);
+            inflate_s.avail_out = static_cast<unsigned int>(2 * size);
+            inflate_s.next_out = reinterpret_cast<Bytef*>(&output[0] + size_uncompressed);
+            int ret = inflate(&inflate_s, Z_FINISH);
+            if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+                std::string error_msg = inflate_s.msg;
+                inflateEnd(&inflate_s);
+                throw std::runtime_error(error_msg);
+            }
+
+            size_uncompressed += (2 * size - inflate_s.avail_out);
+        } while (inflate_s.avail_out == 0);
+        inflateEnd(&inflate_s);
+        output.resize(size_uncompressed);
+    }
+};
+
+inline std::string decompress(const char* data, std::size_t size) {
+    Decompressor decomp;
+    std::string output;
+    decomp.decompress(output, data, size);
+    return output;
+}
+
+}  // namespace gzip

hikyuu_cpp/hikyuu_server/gzip/utils.hpp

@@ -0,0 +1,22 @@
+#include <cstdlib>
+
+namespace gzip {
+
+// These live in gzip.hpp because it doesnt need to use deps.
+// Otherwise, they would need to live in impl files if these methods used
+// zlib structures or functions like inflate/deflate)
+inline bool is_compressed(const char* data, std::size_t size)
+{
+    return size > 2 &&
+           (
+               // zlib
+               (
+                   static_cast<uint8_t>(data[0]) == 0x78 &&
+                   (static_cast<uint8_t>(data[1]) == 0x9C ||
+                    static_cast<uint8_t>(data[1]) == 0x01 ||
+                    static_cast<uint8_t>(data[1]) == 0xDA ||
+                    static_cast<uint8_t>(data[1]) == 0x5E)) ||
+               // gzip
+               (static_cast<uint8_t>(data[0]) == 0x1F && static_cast<uint8_t>(data[1]) == 0x8B));
+}
+} // namespace gzip

hikyuu_cpp/hikyuu_server/gzip/version.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+/// The major version number
+#define GZIP_VERSION_MAJOR 1
+
+/// The minor version number
+#define GZIP_VERSION_MINOR 0
+
+/// The patch number
+#define GZIP_VERSION_PATCH 0
+
+/// The complete version number
+#define GZIP_VERSION_CODE (GZIP_VERSION_MAJOR * 10000 + GZIP_VERSION_MINOR * 100 + GZIP_VERSION_PATCH)
+
+/// Version number as string
+#define GZIP_VERSION_STRING "1.0.0"

hikyuu_cpp/hikyuu_server/http/HttpError.h

@@ -103,43 +103,43 @@ protected:
 };
 #endif /* #ifdef __clang__ */
 
-#define HANDLE_THROW(http_status, ...) \
-    { throw HttpError(http_status, fmt::format(__VA_ARGS__)); }
-
-#define HANDLE_CHECK(expr, http_status, ...)                        \
-    {                                                               \
-        if (!expr) {                                                \
-            throw HttpError(http_status, fmt::format(__VA_ARGS__)); \
-        }                                                           \
-    }
-
-#define HANDLE_CHECK_THROW(expr, error) \
-    {                                   \
-        if (!expr) {                    \
-            throw error;                \
-        }                               \
-    }
-
+/**
+ * Http 400: 请求参数错误
+ */
 class HttpBadRequestError : public HttpError {
 public:
     HttpBadRequestError() : HttpError("HttpBadRequestError") {}
     HttpBadRequestError(int errcode, const char* msg)
     : HttpError("HttpBadRequestError", NNG_HTTP_STATUS_BAD_REQUEST, errcode, msg) {}
-
     HttpBadRequestError(int errcode, const std::string& msg)
     : HttpError("HttpBadRequestError", NNG_HTTP_STATUS_BAD_REQUEST, errcode, msg) {}
 };
 
+/**
+ * Http 401: 当前请求需要用户验证
+ */
 class HttpUnauthorizedError : public HttpError {
 public:
     HttpUnauthorizedError() : HttpError("HttpUnauthorizedError") {}
     HttpUnauthorizedError(int errcode, const char* msg)
     : HttpError("HttpUnauthorizedError", NNG_HTTP_STATUS_UNAUTHORIZED, errcode, msg) {}
-
     HttpUnauthorizedError(int errcode, const std::string& msg)
     : HttpError("HttpUnauthorizedError", NNG_HTTP_STATUS_UNAUTHORIZED, errcode, msg) {}
 };
 
+class HttpNotAcceptableError : public HttpError {
+public:
+    enum NotAcceptableErrorCode {
+        UNSUPPORT_CONTENT_ENCODING = 4060001,  // 不支持的内容编码
+    };
+
+    HttpNotAcceptableError() : HttpError("HttpNotAcceptableError") {}
+    HttpNotAcceptableError(int errcode, const char* msg)
+    : HttpError("HttpNotAcceptableError", NNG_HTTP_STATUS_NOT_ACCEPTABLE, errcode, msg) {}
+    HttpNotAcceptableError(int errcode, const std::string& msg)
+    : HttpError("HttpNotAcceptableError", NNG_HTTP_STATUS_NOT_ACCEPTABLE, errcode, msg) {}
+};
+
 /**
  * 公共错误码
  */

hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp

@@ -7,6 +7,9 @@
 
 #include <string_view>
 #include <hikyuu/utilities/arithmetic.h>
+#include "gzip/compress.hpp"
+#include "gzip/decompress.hpp"
+#include "gzip/utils.hpp"
 #include "url.h"
 #include "HttpHandle.h"
 
@@ -77,37 +80,20 @@ void HttpHandle::trace() {
 
 std::string HttpHandle::getTraceInfo() {
     std::ostringstream out;
-    const char* url = nng_http_req_get_uri(m_nng_req);
-    if (url) {
-        out << url << std::endl;
-        out << "    url: " << url << std::endl;
-    } else {
-        out << "url: null" << std::endl;
-        out << "    url: null" << std::endl;
-    }
+    std::string url = getReqUrl();
+    out << url << std::endl;
+    out << "    url: " << url << std::endl;
 
     std::string traceid = getReqHeader("traceid");
     if (!traceid.empty()) {
         out << "    traceid: " << traceid << std::endl;
     }
 
-    char* data;
-    size_t len = 0;
-    getReqData((void**)&data, &len);
-    out << "    request: ";
-    if (data) {
-        out << std::string_view(data, len) << std::endl;
-    } else {
-        out << "null" << std::endl;
-    }
+    std::string body = getReqData();
+    out << "    request: " << body << std::endl;
 
-    out << "    response: ";
-    nng_http_res_get_data(m_nng_res, (void**)&data, &len);
-    if (data) {
-        out << data << std::endl;
-    } else {
-        out << "null" << std::endl;
-    }
+    body = getResData();
+    out << "    response: " << body << std::endl;
 
     return out.str();
 }
@@ -130,6 +116,15 @@ void HttpHandle::processException(int http_status, int errcode, std::string_view
     }
 }
 
+std::string HttpHandle::getReqUrl() const {
+    std::string result;
+    const char* url = nng_http_req_get_uri(m_nng_req);
+    if (url) {
+        result = std::string(url);
+    }
+    return result;
+}
+
 std::string HttpHandle::getReqHeader(const char* name) const {
     std::string result;
     const char* head = nng_http_req_get_header(m_nng_req, name);
@@ -140,23 +135,59 @@ std::string HttpHandle::getReqHeader(const char* name) const {
 }
 
 std::string HttpHandle::getReqData() {
+    std::string result;
     void* data = nullptr;
     size_t len = 0;
     nng_http_req_get_data(m_nng_req, &data, &len);
-    return data ? std::string((char*)data, len) : std::string();
+    if (!data || len == 0) {
+        return result;
+    }
+
+    std::string encoding = getReqHeader("Content-Encoding");
+    if (encoding.empty()) {
+        result = std::string((char*)data, len);
+
+    } else if (encoding == "gzip") {
+        gzip::Decompressor decomp;
+        decomp.decompress(result, (char*)data, len);
+
+    } else {
+        throw HttpNotAcceptableError(
+          HttpNotAcceptableError::UNSUPPORT_CONTENT_ENCODING,
+          fmt::format(_ctr("HttpHandle", "Unsupported Content-Encoding format: {}"), encoding));
+    }
+
+    return result;
+}
+
+std::string HttpHandle::getResData() const {
+    std::string result;
+    char* data = nullptr;
+    size_t len = 0;
+    nng_http_res_get_data(m_nng_res, (void**)&data, &len);
+    if (!data || len == 0) {
+        return result;
+    }
+
+    if (!gzip::is_compressed(data, len)) {
+        result = std::string(data, len);
+        return result;
+    }
+
+    gzip::Decompressor decomp;
+    decomp.decompress(result, data, len);
+    return result;
 }
 
 json HttpHandle::getReqJson() {
-    void* data;
-    size_t len;
-    nng_http_req_get_data(m_nng_req, &data, &len);
+    std::string data = getReqData();
     json result;
     try {
-        if (data) {
-            result = json::parse(std::string_view((char*)data, len));
+        if (!data.empty()) {
+            result = json::parse(data);
         }
     } catch (json::exception& e) {
-        LOG_ERROR("Failed parse json: {}", (const char*)data);
+        LOG_ERROR("Failed parse json: {}", data);
         throw HttpBadRequestError(BadRequestErrorCode::INVALID_JSON_REQUEST, e.what());
     }
     return result;

hikyuu_cpp/hikyuu_server/http/HttpHandle.h

@@ -58,6 +58,9 @@ public:
         m_filters.push_back(filter);
     }
 
+    /** 获取请求的 url */
+    std::string getReqUrl() const;
+
     /**
      * 获取请求头部信息
      * @param name 头部信息名称
@@ -74,23 +77,13 @@ public:
         return getReqHeader(name.c_str());
     }
 
-    /**
-     * 获取请求数据
-     * @param[out] data 请求中的数据起始地址，无数据时返回 NULL
-     * @param[out] len 请求中的数据长度
-     * @note 请求中无数据时, len返回的长度可能不为0
-     */
-    void getReqData(void **data, size_t *len) {
-        nng_http_req_get_data(m_nng_req, data, len);
-    }
-
+    /** 根据 Content-Encoding 进行解码，返回解码后的请求数据 */
     std::string getReqData();
 
+    /** 返回请求的 json 数据，如无法解析为json，将抛出异常*/
     json getReqJson();
 
-    /**
-     * 请求的 ulr 中是否包含 query 参数
-     */
+    /** 判断请求的 ulr 中是否包含 query 参数 */
     bool haveQueryParams();
 
     typedef std::unordered_map<std::string, std::string> QueryParams;
@@ -126,6 +119,9 @@ public:
         setResData(data.dump());
     }
 
+    /** 获取当前的相应数据 */
+    std::string getResData() const;
+
     /**
      * 从 Accept-Language 获取第一个语言类型
      * @note 非严格 html 协议，仅返回排在最前面的语言类型

hikyuu_cpp/hikyuu_server/xmake.lua

@@ -1,7 +1,7 @@
 target("hkuserver")
     set_kind("binary")
     
-    add_packages("fmt", "spdlog", "flatbuffers", "nng", "nlohmann_json", "sqlite3")
+    add_packages("fmt", "spdlog", "flatbuffers", "nng", "nlohmann_json", "sqlite3", "zlib")
     add_deps("hikyuu")
 
     add_includedirs(".")

xmake.lua

@@ -42,6 +42,7 @@ add_requires("flatbuffers", {system=false, configs = {vs_runtime="MD"}})
 add_requires("nng", {system=false, configs = {vs_runtime="MD", cxflags="-fPIC"}})
 add_requires("nlohmann_json", {system=false})
 add_requires("cpp-httplib", {system=false})
+add_requires("zlib", {system=false})
 
 if is_plat("linux") and linuxos.name() == "ubuntu" then
     add_requires("apt::libhdf5-dev", "apt::libmysqlclient-dev", "apt::libsqlite3-dev")