From 2d56004da614aedd47fa005b8cc41f2009f941a0 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Wed, 9 Jun 2021 02:11:42 +0800 Subject: [PATCH] httpserver add gzip (continue) --- hikyuu_cpp/hikyuu/utilities/arithmetic.h | 41 ++++++++ hikyuu_cpp/hikyuu_server/gzip/compress.hpp | 102 +++++++++++++++++++ hikyuu_cpp/hikyuu_server/gzip/config.hpp | 5 + hikyuu_cpp/hikyuu_server/gzip/decompress.hpp | 100 ++++++++++++++++++ hikyuu_cpp/hikyuu_server/gzip/utils.hpp | 22 ++++ hikyuu_cpp/hikyuu_server/gzip/version.hpp | 16 +++ hikyuu_cpp/hikyuu_server/http/HttpError.h | 38 +++---- hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp | 93 +++++++++++------ hikyuu_cpp/hikyuu_server/http/HttpHandle.h | 22 ++-- hikyuu_cpp/hikyuu_server/xmake.lua | 2 +- xmake.lua | 1 + 11 files changed, 378 insertions(+), 64 deletions(-) create mode 100644 hikyuu_cpp/hikyuu_server/gzip/compress.hpp create mode 100644 hikyuu_cpp/hikyuu_server/gzip/config.hpp create mode 100644 hikyuu_cpp/hikyuu_server/gzip/decompress.hpp create mode 100644 hikyuu_cpp/hikyuu_server/gzip/utils.hpp create mode 100644 hikyuu_cpp/hikyuu_server/gzip/version.hpp diff --git a/hikyuu_cpp/hikyuu/utilities/arithmetic.h b/hikyuu_cpp/hikyuu/utilities/arithmetic.h index 23954964..af9c092f 100644 --- a/hikyuu_cpp/hikyuu/utilities/arithmetic.h +++ b/hikyuu_cpp/hikyuu/utilities/arithmetic.h @@ -12,7 +12,9 @@ #define HIKYUU_UTILITIES_ARITHMETIC_H #include +#include #include +#include #include namespace hku { @@ -45,6 +47,45 @@ inline void trim(std::string& s) { s.erase(s.find_last_not_of(" ") + 1); } +/** + * 分割字符串 + * @param str 待封的字符串 + * @param c 分割符 + */ +inline std::vector split(const std::string& str, char c) { + std::vector result; + std::string_view view(str); + size_t prepos = 0; + size_t pos = view.find_first_of(c); + while (pos != std::string::npos) { + result.emplace_back(view.substr(prepos, pos - prepos)); + prepos = pos + 1; + pos = view.find_first_of(c, prepos); + } + + result.emplace_back(view.substr(prepos)); + return result; +} + +/** + * 字符串分割 + * @param view 待分割的 string_view + * @param c 分割符 + */ +inline std::vector split(const std::string_view& view, char c) { + std::vector result; + size_t prepos = 0; + size_t pos = view.find_first_of(c); + while (pos != std::string::npos) { + result.emplace_back(view.substr(prepos, pos - prepos)); + prepos = pos + 1; + pos = view.find_first_of(c, prepos); + } + + result.emplace_back(view.substr(prepos)); + return result; +} + /** @} */ } /* namespace hku */ diff --git a/hikyuu_cpp/hikyuu_server/gzip/compress.hpp b/hikyuu_cpp/hikyuu_server/gzip/compress.hpp new file mode 100644 index 00000000..481209d7 --- /dev/null +++ b/hikyuu_cpp/hikyuu_server/gzip/compress.hpp @@ -0,0 +1,102 @@ +#include + +// zlib +#include + +// std +#include +#include +#include + +namespace gzip { + +class Compressor { + std::size_t max_; + int level_; + +public: + Compressor(int level = Z_DEFAULT_COMPRESSION, + std::size_t max_bytes = + 2000000000) // by default refuse operation if uncompressed data is > 2GB + : max_(max_bytes), level_(level) {} + + template + void compress(InputType& output, const char* data, std::size_t size) const { +#ifdef DEBUG + // Verify if size input will fit into unsigned int, type used for zlib's avail_in + if (size > std::numeric_limits::max()) { + throw std::runtime_error("size arg is too large to fit into unsigned int type"); + } +#endif + if (size > max_) { + throw std::runtime_error("size may use more memory than intended when decompressing"); + } + + z_stream deflate_s; + deflate_s.zalloc = Z_NULL; + deflate_s.zfree = Z_NULL; + deflate_s.opaque = Z_NULL; + deflate_s.avail_in = 0; + deflate_s.next_in = Z_NULL; + + // The windowBits parameter is the base two logarithm of the window size (the size of the + // history buffer). It should be in the range 8..15 for this version of the library. Larger + // values of this parameter result in better compression at the expense of memory usage. + // This range of values also changes the decoding type: + // -8 to -15 for raw deflate + // 8 to 15 for zlib + // (8 to 15) + 16 for gzip + // (8 to 15) + 32 to automatically detect gzip/zlib header (decompression/inflate only) + constexpr int window_bits = 15 + 16; // gzip with windowbits of 15 + + constexpr int mem_level = 8; + // The memory requirements for deflate are (in bytes): + // (1 << (window_bits+2)) + (1 << (mem_level+9)) + // with a default value of 8 for mem_level and our window_bits of 15 + // this is 128Kb +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#endif + if (deflateInit2(&deflate_s, level_, Z_DEFLATED, window_bits, mem_level, + Z_DEFAULT_STRATEGY) != Z_OK) { + throw std::runtime_error("deflate init failed"); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + deflate_s.next_in = reinterpret_cast(data); + deflate_s.avail_in = static_cast(size); + + std::size_t size_compressed = 0; + do { + size_t increase = size / 2 + 1024; + if (output.size() < (size_compressed + increase)) { + output.resize(size_compressed + increase); + } + // There is no way we see that "increase" would not fit in an unsigned int, + // hence we use static cast here to avoid -Wshorten-64-to-32 error + deflate_s.avail_out = static_cast(increase); + deflate_s.next_out = reinterpret_cast((&output[0] + size_compressed)); + // From http://www.zlib.net/zlib_how.html + // "deflate() has a return value that can indicate errors, yet we do not check it here. + // Why not? Well, it turns out that deflate() can do no wrong here." + // Basically only possible error is from deflateInit not working properly + deflate(&deflate_s, Z_FINISH); + size_compressed += (increase - deflate_s.avail_out); + } while (deflate_s.avail_out == 0); + + deflateEnd(&deflate_s); + output.resize(size_compressed); + } +}; + +inline std::string compress(const char* data, std::size_t size, int level = Z_DEFAULT_COMPRESSION) { + Compressor comp(level); + std::string output; + comp.compress(output, data, size); + return output; +} + +} // namespace gzip diff --git a/hikyuu_cpp/hikyuu_server/gzip/config.hpp b/hikyuu_cpp/hikyuu_server/gzip/config.hpp new file mode 100644 index 00000000..21f34a0b --- /dev/null +++ b/hikyuu_cpp/hikyuu_server/gzip/config.hpp @@ -0,0 +1,5 @@ +#pragma once + +#ifndef ZLIB_CONST +#define ZLIB_CONST +#endif \ No newline at end of file diff --git a/hikyuu_cpp/hikyuu_server/gzip/decompress.hpp b/hikyuu_cpp/hikyuu_server/gzip/decompress.hpp new file mode 100644 index 00000000..5c750b7a --- /dev/null +++ b/hikyuu_cpp/hikyuu_server/gzip/decompress.hpp @@ -0,0 +1,100 @@ +#include + +// zlib +#include + +// std +#include +#include +#include + +namespace gzip { + +class Decompressor { + std::size_t max_; + +public: + Decompressor(std::size_t max_bytes = + 1000000000) // by default refuse operation if compressed data is > 1GB + : max_(max_bytes) {} + + template + void decompress(OutputType& output, const char* data, std::size_t size) const { + z_stream inflate_s; + + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + + // The windowBits parameter is the base two logarithm of the window size (the size of the + // history buffer). It should be in the range 8..15 for this version of the library. Larger + // values of this parameter result in better compression at the expense of memory usage. + // This range of values also changes the decoding type: + // -8 to -15 for raw deflate + // 8 to 15 for zlib + // (8 to 15) + 16 for gzip + // (8 to 15) + 32 to automatically detect gzip/zlib header + constexpr int window_bits = 15 + 32; // auto with windowbits of 15 + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#endif + if (inflateInit2(&inflate_s, window_bits) != Z_OK) { + throw std::runtime_error("inflate init failed"); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + inflate_s.next_in = reinterpret_cast(data); + +#ifdef DEBUG + // Verify if size (long type) input will fit into unsigned int, type used for zlib's + // avail_in + std::uint64_t size_64 = size * 2; + if (size_64 > std::numeric_limits::max()) { + inflateEnd(&inflate_s); + throw std::runtime_error("size arg is too large to fit into unsigned int type x2"); + } +#endif + if (size > max_ || (size * 2) > max_) { + inflateEnd(&inflate_s); + throw std::runtime_error("size may use more memory than intended when decompressing"); + } + inflate_s.avail_in = static_cast(size); + std::size_t size_uncompressed = 0; + do { + std::size_t resize_to = size_uncompressed + 2 * size; + if (resize_to > max_) { + inflateEnd(&inflate_s); + throw std::runtime_error( + "size of output string will use more memory then intended when decompressing"); + } + output.resize(resize_to); + inflate_s.avail_out = static_cast(2 * size); + inflate_s.next_out = reinterpret_cast(&output[0] + size_uncompressed); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + std::string error_msg = inflate_s.msg; + inflateEnd(&inflate_s); + throw std::runtime_error(error_msg); + } + + size_uncompressed += (2 * size - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(size_uncompressed); + } +}; + +inline std::string decompress(const char* data, std::size_t size) { + Decompressor decomp; + std::string output; + decomp.decompress(output, data, size); + return output; +} + +} // namespace gzip diff --git a/hikyuu_cpp/hikyuu_server/gzip/utils.hpp b/hikyuu_cpp/hikyuu_server/gzip/utils.hpp new file mode 100644 index 00000000..db123d10 --- /dev/null +++ b/hikyuu_cpp/hikyuu_server/gzip/utils.hpp @@ -0,0 +1,22 @@ +#include + +namespace gzip { + +// These live in gzip.hpp because it doesnt need to use deps. +// Otherwise, they would need to live in impl files if these methods used +// zlib structures or functions like inflate/deflate) +inline bool is_compressed(const char* data, std::size_t size) +{ + return size > 2 && + ( + // zlib + ( + static_cast(data[0]) == 0x78 && + (static_cast(data[1]) == 0x9C || + static_cast(data[1]) == 0x01 || + static_cast(data[1]) == 0xDA || + static_cast(data[1]) == 0x5E)) || + // gzip + (static_cast(data[0]) == 0x1F && static_cast(data[1]) == 0x8B)); +} +} // namespace gzip diff --git a/hikyuu_cpp/hikyuu_server/gzip/version.hpp b/hikyuu_cpp/hikyuu_server/gzip/version.hpp new file mode 100644 index 00000000..47af692e --- /dev/null +++ b/hikyuu_cpp/hikyuu_server/gzip/version.hpp @@ -0,0 +1,16 @@ +#pragma once + +/// The major version number +#define GZIP_VERSION_MAJOR 1 + +/// The minor version number +#define GZIP_VERSION_MINOR 0 + +/// The patch number +#define GZIP_VERSION_PATCH 0 + +/// The complete version number +#define GZIP_VERSION_CODE (GZIP_VERSION_MAJOR * 10000 + GZIP_VERSION_MINOR * 100 + GZIP_VERSION_PATCH) + +/// Version number as string +#define GZIP_VERSION_STRING "1.0.0" \ No newline at end of file diff --git a/hikyuu_cpp/hikyuu_server/http/HttpError.h b/hikyuu_cpp/hikyuu_server/http/HttpError.h index d2295c02..b13576b3 100644 --- a/hikyuu_cpp/hikyuu_server/http/HttpError.h +++ b/hikyuu_cpp/hikyuu_server/http/HttpError.h @@ -103,43 +103,43 @@ protected: }; #endif /* #ifdef __clang__ */ -#define HANDLE_THROW(http_status, ...) \ - { throw HttpError(http_status, fmt::format(__VA_ARGS__)); } - -#define HANDLE_CHECK(expr, http_status, ...) \ - { \ - if (!expr) { \ - throw HttpError(http_status, fmt::format(__VA_ARGS__)); \ - } \ - } - -#define HANDLE_CHECK_THROW(expr, error) \ - { \ - if (!expr) { \ - throw error; \ - } \ - } - +/** + * Http 400: 请求参数错误 + */ class HttpBadRequestError : public HttpError { public: HttpBadRequestError() : HttpError("HttpBadRequestError") {} HttpBadRequestError(int errcode, const char* msg) : HttpError("HttpBadRequestError", NNG_HTTP_STATUS_BAD_REQUEST, errcode, msg) {} - HttpBadRequestError(int errcode, const std::string& msg) : HttpError("HttpBadRequestError", NNG_HTTP_STATUS_BAD_REQUEST, errcode, msg) {} }; +/** + * Http 401: 当前请求需要用户验证 + */ class HttpUnauthorizedError : public HttpError { public: HttpUnauthorizedError() : HttpError("HttpUnauthorizedError") {} HttpUnauthorizedError(int errcode, const char* msg) : HttpError("HttpUnauthorizedError", NNG_HTTP_STATUS_UNAUTHORIZED, errcode, msg) {} - HttpUnauthorizedError(int errcode, const std::string& msg) : HttpError("HttpUnauthorizedError", NNG_HTTP_STATUS_UNAUTHORIZED, errcode, msg) {} }; +class HttpNotAcceptableError : public HttpError { +public: + enum NotAcceptableErrorCode { + UNSUPPORT_CONTENT_ENCODING = 4060001, // 不支持的内容编码 + }; + + HttpNotAcceptableError() : HttpError("HttpNotAcceptableError") {} + HttpNotAcceptableError(int errcode, const char* msg) + : HttpError("HttpNotAcceptableError", NNG_HTTP_STATUS_NOT_ACCEPTABLE, errcode, msg) {} + HttpNotAcceptableError(int errcode, const std::string& msg) + : HttpError("HttpNotAcceptableError", NNG_HTTP_STATUS_NOT_ACCEPTABLE, errcode, msg) {} +}; + /** * 公共错误码 */ diff --git a/hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp b/hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp index f7ad960a..4fa9dbd0 100644 --- a/hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp +++ b/hikyuu_cpp/hikyuu_server/http/HttpHandle.cpp @@ -7,6 +7,9 @@ #include #include +#include "gzip/compress.hpp" +#include "gzip/decompress.hpp" +#include "gzip/utils.hpp" #include "url.h" #include "HttpHandle.h" @@ -77,37 +80,20 @@ void HttpHandle::trace() { std::string HttpHandle::getTraceInfo() { std::ostringstream out; - const char* url = nng_http_req_get_uri(m_nng_req); - if (url) { - out << url << std::endl; - out << " url: " << url << std::endl; - } else { - out << "url: null" << std::endl; - out << " url: null" << std::endl; - } + std::string url = getReqUrl(); + out << url << std::endl; + out << " url: " << url << std::endl; std::string traceid = getReqHeader("traceid"); if (!traceid.empty()) { out << " traceid: " << traceid << std::endl; } - char* data; - size_t len = 0; - getReqData((void**)&data, &len); - out << " request: "; - if (data) { - out << std::string_view(data, len) << std::endl; - } else { - out << "null" << std::endl; - } + std::string body = getReqData(); + out << " request: " << body << std::endl; - out << " response: "; - nng_http_res_get_data(m_nng_res, (void**)&data, &len); - if (data) { - out << data << std::endl; - } else { - out << "null" << std::endl; - } + body = getResData(); + out << " response: " << body << std::endl; return out.str(); } @@ -130,6 +116,15 @@ void HttpHandle::processException(int http_status, int errcode, std::string_view } } +std::string HttpHandle::getReqUrl() const { + std::string result; + const char* url = nng_http_req_get_uri(m_nng_req); + if (url) { + result = std::string(url); + } + return result; +} + std::string HttpHandle::getReqHeader(const char* name) const { std::string result; const char* head = nng_http_req_get_header(m_nng_req, name); @@ -140,23 +135,59 @@ std::string HttpHandle::getReqHeader(const char* name) const { } std::string HttpHandle::getReqData() { + std::string result; void* data = nullptr; size_t len = 0; nng_http_req_get_data(m_nng_req, &data, &len); - return data ? std::string((char*)data, len) : std::string(); + if (!data || len == 0) { + return result; + } + + std::string encoding = getReqHeader("Content-Encoding"); + if (encoding.empty()) { + result = std::string((char*)data, len); + + } else if (encoding == "gzip") { + gzip::Decompressor decomp; + decomp.decompress(result, (char*)data, len); + + } else { + throw HttpNotAcceptableError( + HttpNotAcceptableError::UNSUPPORT_CONTENT_ENCODING, + fmt::format(_ctr("HttpHandle", "Unsupported Content-Encoding format: {}"), encoding)); + } + + return result; +} + +std::string HttpHandle::getResData() const { + std::string result; + char* data = nullptr; + size_t len = 0; + nng_http_res_get_data(m_nng_res, (void**)&data, &len); + if (!data || len == 0) { + return result; + } + + if (!gzip::is_compressed(data, len)) { + result = std::string(data, len); + return result; + } + + gzip::Decompressor decomp; + decomp.decompress(result, data, len); + return result; } json HttpHandle::getReqJson() { - void* data; - size_t len; - nng_http_req_get_data(m_nng_req, &data, &len); + std::string data = getReqData(); json result; try { - if (data) { - result = json::parse(std::string_view((char*)data, len)); + if (!data.empty()) { + result = json::parse(data); } } catch (json::exception& e) { - LOG_ERROR("Failed parse json: {}", (const char*)data); + LOG_ERROR("Failed parse json: {}", data); throw HttpBadRequestError(BadRequestErrorCode::INVALID_JSON_REQUEST, e.what()); } return result; diff --git a/hikyuu_cpp/hikyuu_server/http/HttpHandle.h b/hikyuu_cpp/hikyuu_server/http/HttpHandle.h index d7d4bacf..108e6339 100644 --- a/hikyuu_cpp/hikyuu_server/http/HttpHandle.h +++ b/hikyuu_cpp/hikyuu_server/http/HttpHandle.h @@ -58,6 +58,9 @@ public: m_filters.push_back(filter); } + /** 获取请求的 url */ + std::string getReqUrl() const; + /** * 获取请求头部信息 * @param name 头部信息名称 @@ -74,23 +77,13 @@ public: return getReqHeader(name.c_str()); } - /** - * 获取请求数据 - * @param[out] data 请求中的数据起始地址，无数据时返回 NULL - * @param[out] len 请求中的数据长度 - * @note 请求中无数据时, len返回的长度可能不为0 - */ - void getReqData(void **data, size_t *len) { - nng_http_req_get_data(m_nng_req, data, len); - } - + /** 根据 Content-Encoding 进行解码，返回解码后的请求数据 */ std::string getReqData(); + /** 返回请求的 json 数据，如无法解析为json，将抛出异常*/ json getReqJson(); - /** - * 请求的 ulr 中是否包含 query 参数 - */ + /** 判断请求的 ulr 中是否包含 query 参数 */ bool haveQueryParams(); typedef std::unordered_map QueryParams; @@ -126,6 +119,9 @@ public: setResData(data.dump()); } + /** 获取当前的相应数据 */ + std::string getResData() const; + /** * 从 Accept-Language 获取第一个语言类型 * @note 非严格 html 协议，仅返回排在最前面的语言类型 diff --git a/hikyuu_cpp/hikyuu_server/xmake.lua b/hikyuu_cpp/hikyuu_server/xmake.lua index 048530d0..946f7b55 100644 --- a/hikyuu_cpp/hikyuu_server/xmake.lua +++ b/hikyuu_cpp/hikyuu_server/xmake.lua @@ -1,7 +1,7 @@ target("hkuserver") set_kind("binary") - add_packages("fmt", "spdlog", "flatbuffers", "nng", "nlohmann_json", "sqlite3") + add_packages("fmt", "spdlog", "flatbuffers", "nng", "nlohmann_json", "sqlite3", "zlib") add_deps("hikyuu") add_includedirs(".") diff --git a/xmake.lua b/xmake.lua index 8a7e75a8..97ef29aa 100644 --- a/xmake.lua +++ b/xmake.lua @@ -42,6 +42,7 @@ add_requires("flatbuffers", {system=false, configs = {vs_runtime="MD"}}) add_requires("nng", {system=false, configs = {vs_runtime="MD", cxflags="-fPIC"}}) add_requires("nlohmann_json", {system=false}) add_requires("cpp-httplib", {system=false}) +add_requires("zlib", {system=false}) if is_plat("linux") and linuxos.name() == "ubuntu" then add_requires("apt::libhdf5-dev", "apt::libmysqlclient-dev", "apt::libsqlite3-dev")