From 5af90429e17e4029e995531dd173f842e66ea507 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Tue, 27 Aug 2024 22:21:52 +0800 Subject: [PATCH 01/21] update --- hikyuu_cpp/demo/demo3.cpp | 114 ++++++++++++++++++++++++++++++++++++++ hikyuu_cpp/demo/xmake.lua | 24 ++++++++ 2 files changed, 138 insertions(+) create mode 100644 hikyuu_cpp/demo/demo3.cpp diff --git a/hikyuu_cpp/demo/demo3.cpp b/hikyuu_cpp/demo/demo3.cpp new file mode 100644 index 00000000..9f5b3327 --- /dev/null +++ b/hikyuu_cpp/demo/demo3.cpp @@ -0,0 +1,114 @@ +// demo.cpp : 定义控制台应用程序的入口点。 +// + +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#endif + +using namespace hku; + +NodeServer server; + +void signal_handle(int signal) { + if (signal == SIGINT || signal == SIGTERM) { + HKU_INFO("Shutdown now ..."); + server.stop(); + exit(0); + } +} + +int main(int argc, char* argv[]) { + std::signal(SIGINT, signal_handle); + std::signal(SIGTERM, signal_handle); + + initLogger(false, "./demo.log"); + +#if defined(_WIN32) + // Windows 下设置控制台程序输出代码页为 UTF8 + auto old_cp = GetConsoleOutputCP(); + SetConsoleOutputCP(CP_UTF8); +#endif + + try { + // 配置文件的位置自行修改 + hikyuu_init("C:\\Users\\admin\\.hikyuu\\hikyuu.ini"); + + // 启动行情接收（只是计算回测可以不需要） + startSpotAgent(true); + + server.setAddr("tcp://0.0.0.0:9201"); + + server.regHandle("market", [](json&& req) { + HKU_INFO("--> req from {}:{}", req["remote_host"].get(), + req["remote_port"].get()); + HKU_ASSERT(req.contains("ktype")); + HKU_ASSERT(req.contains("codes")); + + string ktype = req["ktype"].get(); + auto& sm = StockManager::instance(); + const auto& param = sm.getPreloadParameter(); + string low_ktype = ktype; + to_lower(low_ktype); + HKU_CHECK(param.tryGet(low_ktype, false), "The ktype: {} is not be preloaded!", + ktype); + + json data; + const auto& jcodes = req["codes"]; + for (auto iter = jcodes.cbegin(); iter != jcodes.cend(); ++iter) { + string market_code = to_string(*iter); + market_code = market_code.substr(1, market_code.size() - 2); + Stock stk = getStock(market_code); + if (stk.isNull()) { + HKU_WARN("Not found stock: {}", market_code); + continue; + } + + KRecordList krecords = + stk.getKRecordList(KQueryByIndex(-1, Null(), ktype)); + if (!krecords.empty()) { + const auto& k = krecords.back(); + json jr; + jr.emplace_back(market_code); + jr.emplace_back(k.datetime.str()); + jr.emplace_back(k.openPrice); + jr.emplace_back(k.highPrice); + jr.emplace_back(k.lowPrice); + jr.emplace_back(k.closePrice); + jr.emplace_back(k.transAmount); + jr.emplace_back(k.transCount); + data.emplace_back(std::move(jr)); + } + } + + json res; + res["data"] = data; + // HKU_INFO("<-- res: {}", to_string(res)); + return res; + }); + + server.start(); + // server.loop(); + while (true) { + std::this_thread::sleep_for(std::chrono::seconds(10)); + } + + } catch (const std::exception& e) { + HKU_ERROR(e.what()); + } catch (...) { + HKU_ERROR("Unknown error!"); + } + + server.stop(); + +#if defined(_WIN32) + SetConsoleOutputCP(old_cp); +#endif + return 0; +} diff --git a/hikyuu_cpp/demo/xmake.lua b/hikyuu_cpp/demo/xmake.lua index 59fb328a..1953b42e 100644 --- a/hikyuu_cpp/demo/xmake.lua +++ b/hikyuu_cpp/demo/xmake.lua @@ -44,3 +44,27 @@ target("demo2") add_files("./demo2.cpp") target_end() + + +target("demo3") + set_kind("binary") + set_default(false) + + add_packages("boost", "spdlog", "fmt", "nng", "nlohmann_json") + add_includedirs("..") + + if is_plat("windows") then + add_cxflags("-wd4267") + add_cxflags("-wd4251") + end + + if is_plat("windows") and get_config("kind") == "shared" then + add_defines("HKU_API=__declspec(dllimport)") + add_defines("HKU_UTILS_API=__declspec(dllimport)") + add_defines("SQLITE_API=__declspec(dllimport)") + end + + add_deps("hikyuu") + + add_files("./demo3.cpp") +target_end() From 3008b2dfbe270a6f2e4609e52e4b4343f0ca6e58 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Wed, 28 Aug 2024 01:53:53 +0800 Subject: [PATCH 02/21] update --- hikyuu_cpp/demo/demo3.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hikyuu_cpp/demo/demo3.cpp b/hikyuu_cpp/demo/demo3.cpp index 9f5b3327..dd270185 100644 --- a/hikyuu_cpp/demo/demo3.cpp +++ b/hikyuu_cpp/demo/demo3.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #if defined(_WIN32) #include @@ -38,7 +39,7 @@ int main(int argc, char* argv[]) { try { // 配置文件的位置自行修改 - hikyuu_init("C:\\Users\\admin\\.hikyuu\\hikyuu.ini"); + hikyuu_init(fmt::format("{}/.hikyuu/hikyuu.ini", getUserDir())); // 启动行情接收（只是计算回测可以不需要） startSpotAgent(true); @@ -61,6 +62,7 @@ int main(int argc, char* argv[]) { json data; const auto& jcodes = req["codes"]; + // HKU_INFO("codes size: {}", jcodes.size()); for (auto iter = jcodes.cbegin(); iter != jcodes.cend(); ++iter) { string market_code = to_string(*iter); market_code = market_code.substr(1, market_code.size() - 2); From cb2a69b73a879790f90a4f321c402a1836ed6546 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Wed, 28 Aug 2024 21:44:52 +0800 Subject: [PATCH 03/21] =?UTF-8?q?=E5=B0=9A=E6=9C=AA=E8=A2=AB=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E8=87=B3=E5=86=85=E5=AD=98=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BF=BD=E7=95=A5=E8=A1=8C=E6=83=85=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp index 1ebca6fd..72fd4b90 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp @@ -37,7 +37,7 @@ static string getSpotMarketCode(const SpotRecord& spot) { static void updateStockDayData(const SpotRecord& spot) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); - HKU_IF_RETURN(stk.isNull(), void()); + HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(KQuery::DAY), void()); // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); KRecord krecord(Datetime(spot.datetime.year(), spot.datetime.month(), spot.datetime.day()), spot.open, spot.high, spot.low, spot.close, spot.amount, spot.volume); @@ -46,7 +46,7 @@ static void updateStockDayData(const SpotRecord& spot) { static void updateStockDayUpData(const SpotRecord& spot, KQuery::KType ktype) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); - HKU_IF_RETURN(stk.isNull(), void()); + HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(ktype), void()); // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); std::function endOfPhase; @@ -120,7 +120,7 @@ static void updateStockDayUpData(const SpotRecord& spot, KQuery::KType ktype) { static void updateStockMinData(const SpotRecord& spot, KQuery::KType ktype) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); - HKU_IF_RETURN(stk.isNull(), void()); + HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(ktype), void()); // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); TimeDelta gap; From 1fb030bf8b0cf0113a5725df89cb173427c40dbd Mon Sep 17 00:00:00 2001 From: fasiondog Date: Wed, 28 Aug 2024 22:12:59 +0800 Subject: [PATCH 04/21] =?UTF-8?q?getDataFromBufferServer=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0K=E7=BA=BF=E6=98=AF=E5=90=A6=E8=A2=AB=E9=A2=84?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/strategy/Strategy.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp index 39898fad..e6967c79 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp @@ -339,6 +339,12 @@ void HKU_API runInStrategy(const PFPtr& pf, const KQuery& query, int adjust_cycl void HKU_API getDataFromBufferServer(const std::string& addr, const StockList& stklist, const KQuery::KType& ktype) { // SPEND_TIME(getDataFromBufferServer); + const auto& preload = StockManager::instance().getPreloadParameter(); + string low_ktype = ktype; + to_lower(low_ktype); + HKU_ERROR_IF_RETURN(!preload.tryGet(low_ktype, false), void(), + "The {} kdata is not preload! Can't update!", low_ktype); + NodeClient client(addr); try { HKU_CHECK(client.dial(), "Failed dial server!"); @@ -358,6 +364,7 @@ void HKU_API getDataFromBufferServer(const std::string& addr, const StockList& s res["msg"].get()); const auto& jdata = res["data"]; + // HKU_INFO("{}", to_string(jdata)); for (auto iter = jdata.cbegin(); iter != jdata.cend(); ++iter) { const auto& r = *iter; try { From 18fe64e2e3903e2bfb8472a14e78fd9ad22b1f0f Mon Sep 17 00:00:00 2001 From: fasiondog Date: Wed, 28 Aug 2024 23:56:57 +0800 Subject: [PATCH 05/21] =?UTF-8?q?=E5=88=86=E9=92=9F=E7=BA=A7=E5=88=AB?= =?UTF-8?q?=E8=A1=8C=E6=83=85=E6=95=B0=E6=8D=AE=E6=9B=B4=E6=96=B0=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E8=BF=87=E6=BB=A4=E6=9C=AC=E8=BA=AB=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E5=88=86=E9=92=9F=E7=BA=A7=E5=88=AB=E6=95=B0=E6=8D=AE=E7=9A=84?= =?UTF-8?q?=E8=AF=81=E5=88=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/Stock.cpp | 3 ++- hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hikyuu_cpp/hikyuu/Stock.cpp b/hikyuu_cpp/hikyuu/Stock.cpp index fb1e8662..100b32c0 100644 --- a/hikyuu_cpp/hikyuu/Stock.cpp +++ b/hikyuu_cpp/hikyuu/Stock.cpp @@ -901,7 +901,8 @@ void Stock::realtimeUpdate(KRecord record, KQuery::KType inktype) { } else if (tmp.datetime < record.datetime) { m_data->pKData[ktype]->push_back(record); } else { - HKU_INFO("Ignore record, datetime < last record.datetime!"); + HKU_INFO("Ignore record, datetime({}) < last record.datetime({})! {} {}", record.datetime, + tmp.datetime, market_code(), inktype); } } diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp index 72fd4b90..6501973f 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp @@ -149,8 +149,13 @@ static void updateStockMinData(const SpotRecord& spot, KQuery::KType ktype) { } Datetime minute = spot.datetime; - minute = minute - (minute - minute.startOfDay()) % gap; - KRecordList klist = stk.getKRecordList(KQuery(minute, minute + gap, ktype)); + Datetime today = minute.startOfDay(); + // 非24小时交易品种，且时间和当天零时相同认为无分钟线级别数据 + HKU_IF_RETURN(stk.type() != STOCKTYPE_CRYPTO && minute == today, void()); + + Datetime start_minute = minute - (minute - today) % gap; + Datetime end_minute = start_minute + gap; + KRecordList klist = stk.getKRecordList(KQuery(start_minute, end_minute, ktype)); price_t sum_amount = 0.0, sum_volume = 0.0; for (const auto& k : klist) { sum_amount += k.transAmount; @@ -160,7 +165,7 @@ static void updateStockMinData(const SpotRecord& spot, KQuery::KType ktype) { price_t amount = spot.amount > sum_amount ? spot.amount - sum_amount : spot.amount; price_t spot_volume = spot.volume * 100; // spot 传过来的是手数 price_t volume = spot_volume > sum_volume ? spot_volume - sum_volume : spot_volume; - KRecord krecord(minute, spot.open, spot.high, spot.low, spot.close, amount, volume); + KRecord krecord(end_minute, spot.open, spot.high, spot.low, spot.close, amount, volume); stk.realtimeUpdate(krecord, ktype); } From 0fc80b8d691f0a9f341dbbf0e29c202dad69d947 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Thu, 29 Aug 2024 00:10:53 +0800 Subject: [PATCH 06/21] update spot_server.py --- hikyuu/gui/spot_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hikyuu/gui/spot_server.py b/hikyuu/gui/spot_server.py index 744445ca..cb70977b 100644 --- a/hikyuu/gui/spot_server.py +++ b/hikyuu/gui/spot_server.py @@ -271,7 +271,9 @@ def collect(server, use_proxy, source, seconds, phase1, phase2, ignore_weekend): sm = StockManager.instance() if source == 'qmt': - stk_list = [s for s in sm if s.valid] + stk_list = [s for s in sm if s.valid and s.type in ( + constant.STOCKTYPE_A, constant.STOCKTYPE_INDEX, constant.STOCKTYPE_ETF, + constant.STOCKTYPE_GEM, constant.STOCKTYPE_START, constant.STOCKTYPE_A_BJ)] else: stk_list = [ stk.market_code.lower() for stk in sm if stk.valid and stk.type in From 1331954b06582728a601dfe8a8ccc8a19ab1e44a Mon Sep 17 00:00:00 2001 From: fasiondog Date: Thu, 29 Aug 2024 01:00:50 +0800 Subject: [PATCH 07/21] =?UTF-8?q?=E8=A1=8C=E6=83=85=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=97=B6=E5=8C=85=E6=8B=AC=E9=97=AD=E5=B8=82?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E7=82=B9=E6=95=B0=E6=8D=AE=EF=BC=8C=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E6=9B=B4=E6=96=B0=E6=97=B6=E5=AF=B9=E5=BC=80=E9=97=AD?= =?UTF-8?q?=E5=B8=82=E7=9A=84=E6=97=B6=E9=97=B4=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/Stock.cpp | 4 ++-- hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hikyuu_cpp/hikyuu/Stock.cpp b/hikyuu_cpp/hikyuu/Stock.cpp index 100b32c0..468b04b1 100644 --- a/hikyuu_cpp/hikyuu/Stock.cpp +++ b/hikyuu_cpp/hikyuu/Stock.cpp @@ -865,11 +865,11 @@ bool Stock::isTransactionTime(Datetime time) { Datetime today = Datetime::today(); Datetime openTime1 = today + market_info.openTime1(); Datetime closeTime1 = today + market_info.closeTime1(); - HKU_IF_RETURN(time >= openTime1 && time < closeTime1, true); // close判断不包括等于 + HKU_IF_RETURN(time >= openTime1 && time <= closeTime1, true); // close判断包括等于 Datetime openTime2 = today + market_info.openTime2(); Datetime closeTime2 = today + market_info.closeTime2(); - return time >= openTime2 && time < closeTime2; + return time >= openTime2 && time <= closeTime2; } void Stock::realtimeUpdate(KRecord record, KQuery::KType inktype) { diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp index 6501973f..80b37c0f 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp @@ -38,7 +38,7 @@ static string getSpotMarketCode(const SpotRecord& spot) { static void updateStockDayData(const SpotRecord& spot) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(KQuery::DAY), void()); - // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); + HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); KRecord krecord(Datetime(spot.datetime.year(), spot.datetime.month(), spot.datetime.day()), spot.open, spot.high, spot.low, spot.close, spot.amount, spot.volume); stk.realtimeUpdate(krecord, KQuery::DAY); @@ -47,7 +47,7 @@ static void updateStockDayData(const SpotRecord& spot) { static void updateStockDayUpData(const SpotRecord& spot, KQuery::KType ktype) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(ktype), void()); - // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); + HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); std::function endOfPhase; std::function startOfPhase; @@ -121,7 +121,7 @@ static void updateStockDayUpData(const SpotRecord& spot, KQuery::KType ktype) { static void updateStockMinData(const SpotRecord& spot, KQuery::KType ktype) { Stock stk = StockManager::instance().getStock(getSpotMarketCode(spot)); HKU_IF_RETURN(stk.isNull() || !stk.isBuffer(ktype), void()); - // HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); + HKU_IF_RETURN(!stk.isTransactionTime(spot.datetime), void()); TimeDelta gap; if (KQuery::MIN == ktype) { From baa9c4b67c676f0fa10d088d81962ae302a3ba13 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Thu, 29 Aug 2024 15:18:18 +0800 Subject: [PATCH 08/21] =?UTF-8?q?=E4=BC=98=E5=8C=96init=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu/interactive.py | 2 - hikyuu_cpp/hikyuu/KQuery.cpp | 2 +- hikyuu_cpp/hikyuu/KQuery.h | 4 +- hikyuu_cpp/hikyuu/StockManager.cpp | 40 +------------------ hikyuu_cpp/hikyuu/StockManager.h | 7 +--- hikyuu_cpp/hikyuu/hikyuu.cpp | 63 ++++++++++++------------------ hikyuu_cpp/hikyuu/hikyuu.h | 4 ++ 7 files changed, 35 insertions(+), 87 deletions(-) diff --git a/hikyuu/interactive.py b/hikyuu/interactive.py index b7918cb7..8bc0ee6b 100644 --- a/hikyuu/interactive.py +++ b/hikyuu/interactive.py @@ -83,8 +83,6 @@ ini.read(config_file, encoding='utf-8') hku_param = Parameter() hku_param["tmpdir"] = ini.get('hikyuu', 'tmpdir') hku_param["datadir"] = ini.get('hikyuu', 'datadir') -if ini.has_option('hikyuu', 'logger'): - hku_param["logger"] = ini['hikyuu']['logger'] if ini.has_option('hikyuu', 'quotation_server'): hku_param["quotation_server"] = ini['hikyuu']['quotation_server'] diff --git a/hikyuu_cpp/hikyuu/KQuery.cpp b/hikyuu_cpp/hikyuu/KQuery.cpp index 1aaa1bdf..9d4ff62a 100644 --- a/hikyuu_cpp/hikyuu/KQuery.cpp +++ b/hikyuu_cpp/hikyuu/KQuery.cpp @@ -55,7 +55,7 @@ static const unordered_map g_ktype2min{ }; // 获取所有的 KType -vector& KQuery::getAllKType() { +const vector& KQuery::getAllKType() { return g_all_ktype; } diff --git a/hikyuu_cpp/hikyuu/KQuery.h b/hikyuu_cpp/hikyuu/KQuery.h index 5b63dcc7..7a9885cf 100644 --- a/hikyuu_cpp/hikyuu/KQuery.h +++ b/hikyuu_cpp/hikyuu/KQuery.h @@ -70,7 +70,7 @@ public: // static const string INVALID_KTYPE; /** 获取所有的 KType */ - static vector& getAllKType(); + static const vector& getAllKType(); static int32_t getKTypeInMin(KType); @@ -93,7 +93,7 @@ public: m_end(Null()), m_queryType(INDEX), m_dataType(DAY), - m_recoverType(NO_RECOVER){}; + m_recoverType(NO_RECOVER) {}; /** * K线查询，范围[start, end) diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index a7db183f..1d25b72b 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -55,44 +55,6 @@ StockManager& StockManager::instance() { return (*m_sm); } -Parameter default_preload_param() { - Parameter param; - param.set("day", true); - param.set("week", false); - param.set("month", false); - param.set("quarter", false); - param.set("halfyear", false); - param.set("year", false); - param.set("min", false); - param.set("min5", false); - param.set("min15", false); - param.set("min30", false); - param.set("min60", false); - param.set("hour2", false); - param.set("ticks", false); - param.set("day_max", 100000); - param.set("week_max", 100000); - param.set("month_max", 100000); - param.set("quarter_max", 100000); - param.set("halfyear_max", 100000); - param.set("year_max", 100000); - param.set("min_max", 5120); - param.set("min5_max", 5120); - param.set("min15_max", 5120); - param.set("min30_max", 5120); - param.set("min60_max", 5120); - param.set("hour2_max", 5120); - param.set("ticks_max", 5120); - return param; -} - -Parameter default_other_param() { - Parameter param; - param.set("tmpdir", "."); - param.set("logger", ""); - return param; -} - void StockManager::init(const Parameter& baseInfoParam, const Parameter& blockParam, const Parameter& kdataParam, const Parameter& preloadParam, const Parameter& hikyuuParam, const StrategyContext& context) { @@ -239,7 +201,7 @@ void StockManager::reload() { continue; } - auto& ktype_list = KQuery::getAllKType(); + const auto& ktype_list = KQuery::getAllKType(); for (auto& ktype : ktype_list) { if (iter->second.isBuffer(ktype)) { tg->submit([=]() mutable { diff --git a/hikyuu_cpp/hikyuu/StockManager.h b/hikyuu_cpp/hikyuu/StockManager.h index c6acd0e8..dccee812 100644 --- a/hikyuu_cpp/hikyuu/StockManager.h +++ b/hikyuu_cpp/hikyuu/StockManager.h @@ -21,9 +21,6 @@ namespace hku { typedef vector MarketList; -Parameter default_preload_param(); -Parameter default_other_param(); - /** * 证券信息统一管理类 * @ingroup StockManage @@ -47,8 +44,8 @@ public: * @param context 策略上下文 */ void init(const Parameter& baseInfoParam, const Parameter& blockParam, - const Parameter& kdataParam, const Parameter& preloadParam = default_preload_param(), - const Parameter& hikyuuParam = default_other_param(), + const Parameter& kdataParam, const Parameter& preloadParam, + const Parameter& hikyuuParam, const StrategyContext& context = StrategyContext({"all"})); /** 重新加载 */ diff --git a/hikyuu_cpp/hikyuu/hikyuu.cpp b/hikyuu_cpp/hikyuu/hikyuu.cpp index a37c9d5d..e8cbb723 100644 --- a/hikyuu_cpp/hikyuu/hikyuu.cpp +++ b/hikyuu_cpp/hikyuu/hikyuu.cpp @@ -21,33 +21,32 @@ static Parameter g_hikyuu_context; void hikyuu_init(const string& config_file_name, bool ignore_preload, const StrategyContext& context) { - IniParser config; - try { - config.read(config_file_name); - - } catch (std::invalid_argument& e) { - HKU_FATAL("Reading configure error! {}", e.what()); - exit(1); - } catch (std::logic_error& e) { - HKU_FATAL("Reading configure error! {}", e.what()); - exit(1); - } catch (...) { - HKU_WARN("Reading configure error! Don't know error!"); - exit(1); + Parameter baseParam, blockParam, kdataParam, preloadParam, hkuParam; + getConfigFromIni(config_file_name, baseParam, blockParam, kdataParam, preloadParam, hkuParam); + if (ignore_preload) { + const auto& ktypes = KQuery::getAllKType(); + for (const auto& ktype : ktypes) { + string low_ktype = ktype; + to_lower(low_ktype); + preloadParam.set(low_ktype, false); + } } - Parameter baseParam, blockParam, kdataParam, preloadParam, hkuParam; + StockManager& sm = StockManager::instance(); + sm.init(baseParam, blockParam, kdataParam, preloadParam, hkuParam, context); +} + +void HKU_API getConfigFromIni(const string& config_file_name, Parameter& baseParam, + Parameter& blockParam, Parameter& kdataParam, Parameter& preloadParam, + Parameter& hkuParam) { + IniParser config; + config.read(config_file_name); hkuParam.set("tmpdir", config.get("hikyuu", "tmpdir", ".")); hkuParam.set("datadir", config.get("hikyuu", "datadir", ".")); hkuParam.set("quotation_server", config.get("hikyuu", "quotation_server", "ipc:///tmp/hikyuu_real.ipc")); - if (!config.hasSection("baseinfo")) { - HKU_FATAL("Missing configure of baseinfo!"); - exit(1); - } - IniParser::StringListPtr option = config.getOptionList("baseinfo"); for (auto iter = option->begin(); iter != option->end(); ++iter) { string value = config.get("baseinfo", *iter); @@ -69,26 +68,14 @@ void hikyuu_init(const string& config_file_name, bool ignore_preload, kdataParam.set(*iter, config.get("kdata", *iter)); } - option = config.getOptionList("preload"); - - for (auto iter = option->begin(); iter != option->end(); ++iter) { - try { - auto pos = (*iter).find("max"); - if (pos == std::string::npos) { - preloadParam.set(*iter, - ignore_preload ? false : config.getBool("preload", *iter)); - } else if (!ignore_preload) { - preloadParam.set(*iter, config.getInt("preload", *iter)); - } - } catch (const std::exception& e) { - HKU_ERROR("proload param ({}) error! {}!", *iter, e.what()); - } catch (...) { - HKU_ERROR("proload param ({})! Unknown error!", *iter); - } + const auto& ktypes = KQuery::getAllKType(); + for (const auto& ktype : ktypes) { + string low_ktype = ktype; + to_lower(low_ktype); + preloadParam.set(low_ktype, config.getBool("preload", low_ktype, "False")); + string num_preload = fmt::format("{}_max", low_ktype); + preloadParam.set(num_preload, config.getInt("preload", num_preload, "4096")); } - - StockManager& sm = StockManager::instance(); - sm.init(baseParam, blockParam, kdataParam, preloadParam, hkuParam, context); } } // namespace hku diff --git a/hikyuu_cpp/hikyuu/hikyuu.h b/hikyuu_cpp/hikyuu/hikyuu.h index 600e0baa..f08808c5 100644 --- a/hikyuu_cpp/hikyuu/hikyuu.h +++ b/hikyuu_cpp/hikyuu/hikyuu.h @@ -35,6 +35,10 @@ namespace hku { void HKU_API hikyuu_init(const string& config_file_name, bool ignore_preload = false, const StrategyContext& context = StrategyContext({"all"})); +void HKU_API getConfigFromIni(const string& config_file_name, Parameter& baseParam, + Parameter& blockParam, Parameter& kdataParam, Parameter& preloadParam, + Parameter& hkuParam); + /** @} */ } // namespace hku From cae2add39e03cf85100b8243022e081818f57ece Mon Sep 17 00:00:00 2001 From: fasiondog Date: Thu, 29 Aug 2024 22:11:18 +0800 Subject: [PATCH 09/21] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E5=B8=82=E5=9C=BA=E9=97=AD=E5=B8=82=E6=97=B6=E9=97=B4=EF=BC=8C?= =?UTF-8?q?=E5=BB=B6=E9=95=BF5=E5=88=86=E9=92=9F=EF=BC=8C=E6=94=BE?= =?UTF-8?q?=E5=9C=A8qmt=E7=AD=89=E6=9C=80=E5=90=8E=E8=A1=8C=E6=83=85?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=9C=A815=E7=82=B9=E4=B9=8B=E5=90=8E?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E5=BB=B6=E8=BF=9F=E6=95=B0=E7=A7=92=E6=89=8D?= =?UTF-8?q?=E6=9C=89=E6=9C=80=E7=BB=88=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu/data/mysql_upgrade/0020.sql | 4 ++++ hikyuu/data/sqlite_upgrade/0021.sql | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 hikyuu/data/mysql_upgrade/0020.sql create mode 100644 hikyuu/data/sqlite_upgrade/0021.sql diff --git a/hikyuu/data/mysql_upgrade/0020.sql b/hikyuu/data/mysql_upgrade/0020.sql new file mode 100644 index 00000000..8504a27a --- /dev/null +++ b/hikyuu/data/mysql_upgrade/0020.sql @@ -0,0 +1,4 @@ +UPDATE `hku_base`.`market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=1; +UPDATE `hku_base`.`market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=2; +UPDATE `hku_base`.`market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=3; +UPDATE `hku_base`.`version` set `version` = 20; \ No newline at end of file diff --git a/hikyuu/data/sqlite_upgrade/0021.sql b/hikyuu/data/sqlite_upgrade/0021.sql new file mode 100644 index 00000000..0af1b2a6 --- /dev/null +++ b/hikyuu/data/sqlite_upgrade/0021.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; +UPDATE `market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=1; +UPDATE `market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=2; +UPDATE `market` SET `closeTime1`=1135, `closeTime2`=1505 WHERE `marketid`=3; +UPDATE `version` set `version` = 21; +COMMIT; \ No newline at end of file From 6c3fd9f0d1761a967865d30bb33fedf818e40005 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 01:27:51 +0800 Subject: [PATCH 10/21] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=85=AC=E5=85=B1?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=B1=A0=E7=BA=BF=E7=A8=8B=E6=95=B0=E5=8F=AF?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/StockManager.cpp | 21 ++++++++++++-------- hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp | 19 +++++++++++------- hikyuu_cpp/hikyuu/hikyuu.cpp | 14 +++++++++++++ hikyuu_cpp/hikyuu/hikyuu.h | 9 +++++++++ hikyuu_cpp/hikyuu/strategy/Strategy.h | 1 + 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 1d25b72b..88d1ded9 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -480,6 +480,7 @@ void StockManager::loadAllHolidays() { } void StockManager::loadAllStockWeights() { + HKU_IF_RETURN(!m_hikyuuParam.tryGet("load_stock_weight", true), void()); HKU_INFO("Loading stock weight..."); if (m_context.isAll()) { auto all_stkweight_dict = m_baseInfoDriver->getAllStockWeightList(); @@ -511,10 +512,12 @@ void StockManager::loadAllZhBond10() { } void StockManager::loadHistoryFinanceField() { - auto fields = m_baseInfoDriver->getHistoryFinanceField(); - for (const auto& field : fields) { - m_field_ix_to_name[field.first - 1] = field.second; - m_field_name_to_ix[field.second] = field.first - 1; + if (m_hikyuuParam.tryGet("load_history_finance", true)) { + auto fields = m_baseInfoDriver->getHistoryFinanceField(); + for (const auto& field : fields) { + m_field_ix_to_name[field.first - 1] = field.second; + m_field_name_to_ix[field.second] = field.first - 1; + } } } @@ -531,10 +534,12 @@ vector> StockManager::getHistoryFinanceAllFields() con } void StockManager::loadHistoryFinance() { - auto* tg = getGlobalTaskGroup(); - std::lock_guard lock1(*m_stockDict_mutex); - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - tg->submit([=]() { iter->second.getHistoryFinance(); }); + if (m_hikyuuParam.tryGet("load_history_finance", true)) { + auto* tg = getGlobalTaskGroup(); + std::lock_guard lock1(*m_stockDict_mutex); + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { + tg->submit([=]() { iter->second.getHistoryFinance(); }); + } } } diff --git a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp b/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp index d7b7a1f3..746452d5 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp +++ b/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp @@ -10,6 +10,7 @@ #include "hikyuu/GlobalInitializer.h" #include "hikyuu/utilities/Log.h" #include "GlobalTaskGroup.h" +#include "hikyuu/StockManager.h" namespace hku { @@ -18,13 +19,17 @@ static TaskGroup* g_threadPool; TaskGroup* getGlobalTaskGroup() { static std::once_flag oc; std::call_once(oc, [&]() { - auto cpu_num = std::thread::hardware_concurrency(); - if (cpu_num >= 4) { - cpu_num -= 2; - } else if (cpu_num > 1) { - cpu_num--; - } - g_threadPool = new TaskGroup(cpu_num); + // auto cpu_num = std::thread::hardware_concurrency(); + // if (cpu_num >= 4) { + // cpu_num -= 2; + // } else if (cpu_num > 1) { + // cpu_num--; + // } + const auto& param = StockManager::instance().getHikyuuParameter(); + size_t worker_num = + param.tryGet("commont_thread_pool_num", std::thread::hardware_concurrency()); + HKU_INFO("work_num: {}", worker_num); + g_threadPool = new TaskGroup(worker_num); }); return g_threadPool; } diff --git a/hikyuu_cpp/hikyuu/hikyuu.cpp b/hikyuu_cpp/hikyuu/hikyuu.cpp index e8cbb723..478c9776 100644 --- a/hikyuu_cpp/hikyuu/hikyuu.cpp +++ b/hikyuu_cpp/hikyuu/hikyuu.cpp @@ -46,6 +46,20 @@ void HKU_API getConfigFromIni(const string& config_file_name, Parameter& basePar hkuParam.set("datadir", config.get("hikyuu", "datadir", ".")); hkuParam.set("quotation_server", config.get("hikyuu", "quotation_server", "ipc:///tmp/hikyuu_real.ipc")); + // 加载权息数据 + hkuParam.set("load_stock_weight", config.getBool("hikyuu", "load_stock_weight", "True")); + + // 加载历史财务数据 + hkuParam.set("load_history_finance", + config.getBool("hikyuu", "load_history_finance", "True")); + + // 公共线程池线程数量 + size_t cpu_num = 2 * std::thread::hardware_concurrency(); + if (cpu_num > 30) { + cpu_num = 30; + } + hkuParam.set("commont_thread_pool_num", config.getInt("hikyuu", "commont_thread_pool_num", + fmt::format("{}", cpu_num))); IniParser::StringListPtr option = config.getOptionList("baseinfo"); for (auto iter = option->begin(); iter != option->end(); ++iter) { diff --git a/hikyuu_cpp/hikyuu/hikyuu.h b/hikyuu_cpp/hikyuu/hikyuu.h index f08808c5..27cb874a 100644 --- a/hikyuu_cpp/hikyuu/hikyuu.h +++ b/hikyuu_cpp/hikyuu/hikyuu.h @@ -35,6 +35,15 @@ namespace hku { void HKU_API hikyuu_init(const string& config_file_name, bool ignore_preload = false, const StrategyContext& context = StrategyContext({"all"})); +/** + * @brief 尝试从 ini 文件获取配置参数 + * @param config_file_name ini 文件名 + * @param baseParam [out] + * @param blockParam [out] + * @param kdataParam [out] + * @param preloadParam [out] + * @param hkuParam [out] + */ void HKU_API getConfigFromIni(const string& config_file_name, Parameter& baseParam, Parameter& blockParam, Parameter& kdataParam, Parameter& preloadParam, Parameter& hkuParam); diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.h b/hikyuu_cpp/hikyuu/strategy/Strategy.h index 756d491a..e59fa451 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.h +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.h @@ -28,6 +28,7 @@ namespace hku { */ class HKU_API Strategy { CLASS_LOGGER_IMP(Strategy) + PARAMETER_SUPPORT public: Strategy(); From 08526c2f180b4b04f9772dc144ade18b4e77326b Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 12:48:41 +0800 Subject: [PATCH 11/21] =?UTF-8?q?=E8=B0=83=E6=95=B4K=E7=BA=BF=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=8A=A0=E8=BD=BD=E7=AD=96=E7=95=A5=EF=BC=8C=E5=85=88?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E5=90=8C=E4=B8=80=E7=B1=BB=E5=9E=8B=E7=9A=84?= =?UTF-8?q?K=E7=BA=BF=EF=BC=8C=E4=BD=BF=E7=94=A8=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=95=B0=E6=8D=AE=E6=B5=8B=E8=AF=95=E6=97=B6?= =?UTF-8?q?=E4=B8=8D=E7=94=A8=E7=AD=89=E5=BE=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/KQuery.cpp | 2 +- hikyuu_cpp/hikyuu/KQuery.h | 2 +- hikyuu_cpp/hikyuu/StockManager.cpp | 56 +++++++++--------------------- 3 files changed, 18 insertions(+), 42 deletions(-) diff --git a/hikyuu_cpp/hikyuu/KQuery.cpp b/hikyuu_cpp/hikyuu/KQuery.cpp index 9d4ff62a..72c683e7 100644 --- a/hikyuu_cpp/hikyuu/KQuery.cpp +++ b/hikyuu_cpp/hikyuu/KQuery.cpp @@ -55,7 +55,7 @@ static const unordered_map g_ktype2min{ }; // 获取所有的 KType -const vector& KQuery::getAllKType() { +const vector& KQuery::getAllKType() { return g_all_ktype; } diff --git a/hikyuu_cpp/hikyuu/KQuery.h b/hikyuu_cpp/hikyuu/KQuery.h index 7a9885cf..3bc301b2 100644 --- a/hikyuu_cpp/hikyuu/KQuery.h +++ b/hikyuu_cpp/hikyuu/KQuery.h @@ -70,7 +70,7 @@ public: // static const string INVALID_KTYPE; /** 获取所有的 KType */ - static const vector& getAllKType(); + static const vector& getAllKType(); static int32_t getKTypeInMin(KType); diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 88d1ded9..1a490ca7 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -139,20 +139,28 @@ void StockManager::setKDataDriver(const KDataDriverConnectPoolPtr& driver) { } void StockManager::loadAllKData() { - const auto& ktypes = KQuery::getAllKType(); + // 按 K 线类型控制加载顺序 + // const auto& ktypes = KQuery::getAllKType(); + vector ktypes{KQuery::DAY, KQuery::MIN, KQuery::WEEK, KQuery::MONTH, + KQuery::QUARTER, KQuery::HALFYEAR, KQuery::YEAR, KQuery::MIN5, + KQuery::MIN15, KQuery::MIN30, KQuery::MIN60, KQuery::MIN3, + KQuery::HOUR2, KQuery::HOUR4, KQuery::HOUR6, KQuery::HOUR12}; + HKU_ASSERT(ktypes.size() == KQuery::getAllKType().size()); + vector low_ktypes; low_ktypes.reserve(ktypes.size()); for (const auto& ktype : ktypes) { auto& back = low_ktypes.emplace_back(ktype); to_lower(back); - HKU_INFO_IF(m_preloadParam.tryGet(back, false), "Preloading all {} kdata to buffer!", + HKU_INFO_IF(m_preloadParam.tryGet(back, false), "Preloading all {} kdata to buffer !", back); } + // 先加载同类K线 auto driver = DataDriverFactory::getKDataDriverPool(m_kdataDriverParam); if (!driver->getPrototype()->canParallelLoad()) { - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - for (size_t i = 0, len = ktypes.size(); i < len; i++) { + for (size_t i = 0, len = ktypes.size(); i < len; i++) { + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { const auto& low_ktype = low_ktypes[i]; if (m_preloadParam.tryGet(low_ktype, false)) { iter->second.loadKDataToBuffer(ktypes[i]); @@ -163,8 +171,8 @@ void StockManager::loadAllKData() { } else { // 异步并行加载 auto* tg = getGlobalTaskGroup(); - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - for (size_t i = 0, len = ktypes.size(); i < len; i++) { + for (size_t i = 0, len = ktypes.size(); i < len; i++) { + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { const auto& low_ktype = low_ktypes[i]; if (m_preloadParam.tryGet(low_ktype, false)) { tg->submit( @@ -179,6 +187,7 @@ void StockManager::reload() { HKU_IF_RETURN(m_initializing, void()); m_initializing = true; + HKU_INFO("start reload ..."); loadAllHolidays(); loadAllMarketInfos(); loadAllStockTypeInfo(); @@ -188,40 +197,7 @@ void StockManager::reload() { loadHistoryFinanceField(); m_blockDriver->load(); - - HKU_INFO("start reload kdata to buffer"); - std::vector can_not_parallel_stk_list; // 记录不支持并行加载的Stock - { - auto* tg = getGlobalTaskGroup(); - std::lock_guard lock(*m_stockDict_mutex); - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - auto driver = iter->second.getKDataDirver(); - if (!driver->getPrototype()->canParallelLoad()) { - can_not_parallel_stk_list.push_back(iter->second); - continue; - } - - const auto& ktype_list = KQuery::getAllKType(); - for (auto& ktype : ktype_list) { - if (iter->second.isBuffer(ktype)) { - tg->submit([=]() mutable { - Stock& stk = iter->second; - stk.loadKDataToBuffer(ktype); - }); - } - } - } - } - - for (auto& stk : can_not_parallel_stk_list) { - const auto& ktype_list = KQuery::getAllKType(); - for (const auto& ktype : ktype_list) { - if (stk.isBuffer(ktype)) { - stk.loadKDataToBuffer(ktype); - } - } - } - + loadAllKData(); loadHistoryFinance(); m_initializing = false; } From 1305c73512f7a0b2b400fadfbecae2a813e731c6 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 12:49:32 +0800 Subject: [PATCH 12/21] =?UTF-8?q?fixed=20reload=E5=90=8E=20TMP=20market=20?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/StockManager.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 1a490ca7..510e1f3e 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -120,11 +120,6 @@ void StockManager::init(const Parameter& baseInfoParam, const Parameter& blockPa initInnerTask(); - // add special Market, for temp csv file - m_marketInfoDict["TMP"] = - MarketInfo("TMP", "Temp Csv file", "temp load from csv file", "000001", Null(), - TimeDelta(0), TimeDelta(0), TimeDelta(0), TimeDelta(0)); - std::chrono::duration sec = std::chrono::system_clock::now() - start_time; HKU_INFO("{:<.2f}s Loaded Data.", sec.count()); m_initializing = false; @@ -436,6 +431,11 @@ void StockManager::loadAllMarketInfos() { to_upper(market); m_marketInfoDict[market] = marketInfo; } + + // add special Market, for temp csv file + m_marketInfoDict["TMP"] = + MarketInfo("TMP", "Temp Csv file", "temp load from csv file", "000001", Null(), + TimeDelta(0), TimeDelta(0), TimeDelta(0), TimeDelta(0)); } void StockManager::loadAllStockTypeInfo() { From 0d5babe00c28afb773140dc14b18de11f3f82b40 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 14:31:08 +0800 Subject: [PATCH 13/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=20StockManager=20?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E9=83=A8=E5=88=86=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/StockManager.cpp | 73 ++++++++++++------------------ hikyuu_cpp/hikyuu/StockManager.h | 4 +- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 510e1f3e..52f1fdf2 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -87,6 +87,22 @@ void StockManager::init(const Parameter& baseInfoParam, const Parameter& blockPa m_baseInfoDriver = DataDriverFactory::getBaseInfoDriver(baseInfoParam); HKU_CHECK(m_baseInfoDriver, "Failed get base info driver!"); + // 获取板块驱动 + m_blockDriver = DataDriverFactory::getBlockDriver(blockParam); + + auto driver = DataDriverFactory::getKDataDriverPool(m_kdataDriverParam); + HKU_CHECK(driver, "driver is null!"); + if (m_kdataDriverParam != driver->getPrototype()->getParameter()) { + m_kdataDriverParam = driver->getPrototype()->getParameter(); + } + + loadData(); + initInnerTask(); + + m_initializing = false; +} + +void StockManager::loadData() { loadAllHolidays(); loadAllMarketInfos(); loadAllStockTypeInfo(); @@ -95,42 +111,16 @@ void StockManager::init(const Parameter& baseInfoParam, const Parameter& blockPa loadAllZhBond10(); loadHistoryFinanceField(); - // 获取板块驱动 - m_blockDriver = DataDriverFactory::getBlockDriver(blockParam); - // 获取K线数据驱动并预加载指定的数据 HKU_INFO("Loading KData..."); std::chrono::system_clock::time_point start_time = std::chrono::system_clock::now(); - auto driver = DataDriverFactory::getKDataDriverPool(m_kdataDriverParam); - HKU_CHECK(driver, "driver is null!"); - if (m_kdataDriverParam != driver->getPrototype()->getParameter()) { - m_kdataDriverParam = driver->getPrototype()->getParameter(); - } - setKDataDriver(driver); - - // 加载 block，须在 stock 的 kdatadriver 被设置之后调用 m_blockDriver->load(); - - // 加载 K 线至缓存 loadAllKData(); - - // 加载历史财务信息 loadHistoryFinance(); - initInnerTask(); - std::chrono::duration sec = std::chrono::system_clock::now() - start_time; HKU_INFO("{:<.2f}s Loaded Data.", sec.count()); - m_initializing = false; -} - -void StockManager::setKDataDriver(const KDataDriverConnectPoolPtr& driver) { - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - if (iter->second.market() == "TMP") - continue; - iter->second.setKDataDriver(driver); - } } void StockManager::loadAllKData() { @@ -183,17 +173,7 @@ void StockManager::reload() { m_initializing = true; HKU_INFO("start reload ..."); - loadAllHolidays(); - loadAllMarketInfos(); - loadAllStockTypeInfo(); - loadAllStocks(); - loadAllStockWeights(); - loadAllZhBond10(); - loadHistoryFinanceField(); - - m_blockDriver->load(); - loadAllKData(); - loadHistoryFinance(); + loadData(); m_initializing = false; } @@ -372,6 +352,8 @@ void StockManager::loadAllStocks() { } } + auto kdriver = DataDriverFactory::getKDataDriverPool(m_kdataDriverParam); + std::lock_guard lock(*m_stockDict_mutex); for (auto& info : stockInfos) { Datetime startDate, endDate; @@ -385,15 +367,17 @@ void StockManager::loadAllStocks() { } catch (...) { endDate = Null(); } - Stock _stock(info.market, info.code, info.name, info.type, info.valid, startDate, endDate, - info.tick, info.tickValue, info.precision, info.minTradeNumber, - info.maxTradeNumber); - string market_code = _stock.market_code(); - ; + + string market_code = fmt::format("{}{}", info.market, info.code); to_upper(market_code); + auto iter = m_stockDict.find(market_code); if (iter == m_stockDict.end()) { - m_stockDict[market_code] = _stock; + Stock _stock(info.market, info.code, info.name, info.type, info.valid, startDate, + endDate, info.tick, info.tickValue, info.precision, info.minTradeNumber, + info.maxTradeNumber); + _stock.setKDataDriver(kdriver); + m_stockDict[market_code] = std::move(_stock); } else { Stock& stock = iter->second; if (!stock.m_data) { @@ -416,6 +400,9 @@ void StockManager::loadAllStocks() { stock.m_data->m_maxTradeNumber = info.maxTradeNumber; stock.m_data->m_history_finance_ready = false; } + if (!stock.getKDataDirver()) { + stock.setKDataDriver(kdriver); + } } } } diff --git a/hikyuu_cpp/hikyuu/StockManager.h b/hikyuu_cpp/hikyuu/StockManager.h index dccee812..f05f2321 100644 --- a/hikyuu_cpp/hikyuu/StockManager.h +++ b/hikyuu_cpp/hikyuu/StockManager.h @@ -225,8 +225,8 @@ public: } private: - /* 设置K线驱动 */ - void setKDataDriver(const KDataDriverConnectPoolPtr&); + /* 加载全部数据 */ + void loadData(); /* 加载 K线数据至缓存 */ void loadAllKData(); From 7654ec246c6aebc46a059976abece421cfc8ec7e Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 18:54:00 +0800 Subject: [PATCH 14/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=EF=BC=9B=E5=8E=BB=E9=99=A4=E5=85=AC=E5=85=B1?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=BB=84=E5=87=8F=E5=B0=91=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=EF=BC=9B=E4=BC=98=E5=8C=96Strategy=EF=BC=8C=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=97=B6=E7=9B=B4=E6=8E=A5=E4=BD=BF=E7=94=A8=E8=87=AA?= =?UTF-8?q?=E5=B7=B1=E6=89=80=E5=9C=A8=E7=BA=BF=E7=A8=8B=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/GlobalInitializer.cpp | 3 - hikyuu_cpp/hikyuu/StockManager.cpp | 65 ++++++++++++------- hikyuu_cpp/hikyuu/StockManager.h | 11 +++- .../hikyuu/data_driver/BaseInfoDriver.cpp | 1 - hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp | 46 ------------- hikyuu_cpp/hikyuu/global/GlobalTaskGroup.h | 44 ------------- hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp | 4 +- .../hikyuu/global/schedule/inner_tasks.cpp | 1 - .../hikyuu/global/schedule/scheduler.cpp | 4 +- hikyuu_cpp/hikyuu/hikyuu.cpp | 8 --- hikyuu_cpp/hikyuu/strategy/Strategy.cpp | 21 ++---- 11 files changed, 59 insertions(+), 149 deletions(-) delete mode 100644 hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp delete mode 100644 hikyuu_cpp/hikyuu/global/GlobalTaskGroup.h diff --git a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp index 0f6e0fcb..5da311f9 100644 --- a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp +++ b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp @@ -25,7 +25,6 @@ #include "hikyuu.h" #include "GlobalInitializer.h" #include "StockManager.h" -#include "global/GlobalTaskGroup.h" #include "global/GlobalSpotAgent.h" #include "global/schedule/scheduler.h" #include "indicator/IndicatorImp.h" @@ -70,7 +69,6 @@ void GlobalInitializer::init() { DataDriverFactory::init(); StockManager::instance(); IndicatorImp::initDynEngine(); - getGlobalSpotAgent(); } void GlobalInitializer::clean() { @@ -87,7 +85,6 @@ void GlobalInitializer::clean() { #endif releaseScheduler(); - releaseGlobalTaskGroup(); releaseGlobalSpotAgent(); IndicatorImp::releaseDynEngine(); diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 52f1fdf2..822e5513 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -18,7 +18,6 @@ #include "hikyuu/utilities/ini_parser/IniParser.h" #include "hikyuu/utilities/thread/ThreadPool.h" #include "StockManager.h" -#include "global/GlobalTaskGroup.h" #include "global/schedule/inner_tasks.h" #include "data_driver/kdata/cvs/KDataTempCsvDriver.h" @@ -33,7 +32,7 @@ void StockManager::quit() { } } -StockManager::StockManager() : m_initializing(false) { +StockManager::StockManager() : m_initializing(false), m_data_ready(false) { m_stockDict_mutex = new std::mutex; m_marketInfoDict_mutex = new std::mutex; m_stockTypeInfo_mutex = new std::mutex; @@ -103,6 +102,9 @@ void StockManager::init(const Parameter& baseInfoParam, const Parameter& blockPa } void StockManager::loadData() { + std::chrono::system_clock::time_point start_time = std::chrono::system_clock::now(); + m_data_ready = false; + loadAllHolidays(); loadAllMarketInfos(); loadAllStockTypeInfo(); @@ -111,13 +113,16 @@ void StockManager::loadData() { loadAllZhBond10(); loadHistoryFinanceField(); + HKU_INFO("Loading block..."); + m_blockDriver->load(); + // 获取K线数据驱动并预加载指定的数据 HKU_INFO("Loading KData..."); - std::chrono::system_clock::time_point start_time = std::chrono::system_clock::now(); - m_blockDriver->load(); + auto driver = DataDriverFactory::getKDataDriverPool(m_kdataDriverParam); + + // 加载K线及历史财务信息 loadAllKData(); - loadHistoryFinance(); std::chrono::duration sec = std::chrono::system_clock::now() - start_time; HKU_INFO("{:<.2f}s Loaded Data.", sec.count()); @@ -153,18 +158,42 @@ void StockManager::loadAllKData() { } } + if (m_hikyuuParam.tryGet("load_history_finance", true)) { + ThreadPool tg; + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { + tg.submit([stk = iter->second]() { stk.getHistoryFinance(); }); + } + tg.join(); + } + + m_data_ready = true; + } else { // 异步并行加载 - auto* tg = getGlobalTaskGroup(); - for (size_t i = 0, len = ktypes.size(); i < len; i++) { - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - const auto& low_ktype = low_ktypes[i]; - if (m_preloadParam.tryGet(low_ktype, false)) { - tg->submit( - [=, ktype = ktypes[i]]() mutable { iter->second.loadKDataToBuffer(ktype); }); + std::thread t = std::thread([this, ktypes, low_ktypes]() { + ThreadPool tg(std::thread::hardware_concurrency()); + for (size_t i = 0, len = ktypes.size(); i < len; i++) { + std::lock_guard lock(*m_stockDict_mutex); + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { + if (m_preloadParam.tryGet(low_ktypes[i], false)) { + tg.submit([stk = iter->second, ktype = std::move(ktypes[i])]() mutable { + stk.loadKDataToBuffer(ktype); + }); + } } } - } + + if (m_hikyuuParam.tryGet("load_history_finance", true)) { + std::lock_guard lock(*m_stockDict_mutex); + for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { + tg.submit([stk = iter->second]() { stk.getHistoryFinance(); }); + } + } + + tg.join(); + m_data_ready = true; + }); + t.detach(); } } @@ -496,14 +525,4 @@ vector> StockManager::getHistoryFinanceAllFields() con return ret; } -void StockManager::loadHistoryFinance() { - if (m_hikyuuParam.tryGet("load_history_finance", true)) { - auto* tg = getGlobalTaskGroup(); - std::lock_guard lock1(*m_stockDict_mutex); - for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - tg->submit([=]() { iter->second.getHistoryFinance(); }); - } - } -} - } // namespace hku diff --git a/hikyuu_cpp/hikyuu/StockManager.h b/hikyuu_cpp/hikyuu/StockManager.h index f05f2321..33a78103 100644 --- a/hikyuu_cpp/hikyuu/StockManager.h +++ b/hikyuu_cpp/hikyuu/StockManager.h @@ -87,6 +87,9 @@ public: /** 获取证券数量 */ size_t size() const; + /** 是否所有数据准备完毕 */ + bool dataReady() const; + /** * 根据"市场简称证券代码"获取对应的证券实例 * @param querystr 格式：“市场简称证券代码”，如"sh000001" @@ -252,15 +255,13 @@ private: /** 加载历史财经字段索引 */ void loadHistoryFinanceField(); - /** 加载历史财务数据 */ - void loadHistoryFinance(); - private: StockManager(); private: static StockManager* m_sm; std::atomic_bool m_initializing; + std::atomic_bool m_data_ready; // 用于指示是否所有数据准备完毕 std::thread::id m_thread_id; // 记录线程id，用于判断Stratege是以独立进程方式还是线程方式运行 string m_tmpdir; string m_datadir; @@ -298,6 +299,10 @@ inline size_t StockManager::size() const { return m_stockDict.size(); } +inline bool StockManager::dataReady() const { + return m_data_ready; +} + inline Stock StockManager::operator[](const string& query) const { return getStock(query); } diff --git a/hikyuu_cpp/hikyuu/data_driver/BaseInfoDriver.cpp b/hikyuu_cpp/hikyuu/data_driver/BaseInfoDriver.cpp index d69a122c..cf08be7c 100644 --- a/hikyuu_cpp/hikyuu/data_driver/BaseInfoDriver.cpp +++ b/hikyuu_cpp/hikyuu/data_driver/BaseInfoDriver.cpp @@ -7,7 +7,6 @@ #include #include "../StockManager.h" -#include "../global/GlobalTaskGroup.h" #include "BaseInfoDriver.h" namespace hku { diff --git a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp b/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp deleted file mode 100644 index 746452d5..00000000 --- a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * GlobalTaskGroup.cpp - * - * Copyright (c) 2019 hikyuu.org - * - * Created on: 2020-4-20 - * Author: fasiondog - */ - -#include "hikyuu/GlobalInitializer.h" -#include "hikyuu/utilities/Log.h" -#include "GlobalTaskGroup.h" -#include "hikyuu/StockManager.h" - -namespace hku { - -static TaskGroup* g_threadPool; - -TaskGroup* getGlobalTaskGroup() { - static std::once_flag oc; - std::call_once(oc, [&]() { - // auto cpu_num = std::thread::hardware_concurrency(); - // if (cpu_num >= 4) { - // cpu_num -= 2; - // } else if (cpu_num > 1) { - // cpu_num--; - // } - const auto& param = StockManager::instance().getHikyuuParameter(); - size_t worker_num = - param.tryGet("commont_thread_pool_num", std::thread::hardware_concurrency()); - HKU_INFO("work_num: {}", worker_num); - g_threadPool = new TaskGroup(worker_num); - }); - return g_threadPool; -} - -void releaseGlobalTaskGroup() { - HKU_TRACE("releaseGlobalTaskGroup"); - if (g_threadPool) { - g_threadPool->stop(); - delete g_threadPool; - g_threadPool = nullptr; - } -} - -} /* namespace hku */ \ No newline at end of file diff --git a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.h b/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.h deleted file mode 100644 index 72ab4bad..00000000 --- a/hikyuu_cpp/hikyuu/global/GlobalTaskGroup.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * GlobalTaskGroup.h - * - * Copyright (c) 2019 hikyuu.org - * - * Created on: 2020-4-20 - * Author: fasiondog - */ - -#pragma once -#ifndef HKU_GLOBAL_TASK_GROUP -#define HKU_GLOBAL_TASK_GROUP - -#include "../utilities/thread/thread.h" - -namespace hku { - -using TaskGroup = ThreadPool; - -/** - * 获取全局线程池任务组 - * @note 请使用 future 获取任务返回 - */ -TaskGroup* getGlobalTaskGroup(); - -template -using task_handle = std::future; - -/** - * 向全局任务池中增加任务 - */ -template -task_handle::type> addTask(FunctionType f) { - return getGlobalTaskGroup()->submit(f); -} - -/** - * 程序退出时释放全局任务组实例，仅内部调用 - */ -void releaseGlobalTaskGroup(); - -} /* namespace hku */ - -#endif /* HKU_GLOBAL_TASK_GROUP */ diff --git a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp index 24552478..7c2eb084 100644 --- a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp @@ -128,7 +128,7 @@ void SpotAgent::parseSpotData(const void* buf, size_t buf_len) { #pragma warning(disable : 4267) #endif - // 更新日线数据 + // 更新K线数据 auto* spot_list = GetSpotList(spot_list_buf); auto* spots = spot_list->spot(); size_t total = spots->size(); @@ -138,7 +138,7 @@ void SpotAgent::parseSpotData(const void* buf, size_t buf_len) { auto spot_record = parseFlatSpot(spot); if (spot_record) { for (const auto& process : m_processList) { - m_process_task_list.push_back(m_tg.submit(ProcessTask(process, *spot_record))); + m_process_task_list.emplace_back(m_tg.submit(ProcessTask(process, *spot_record))); } } } diff --git a/hikyuu_cpp/hikyuu/global/schedule/inner_tasks.cpp b/hikyuu_cpp/hikyuu/global/schedule/inner_tasks.cpp index a9622c1e..433bcbd0 100644 --- a/hikyuu_cpp/hikyuu/global/schedule/inner_tasks.cpp +++ b/hikyuu_cpp/hikyuu/global/schedule/inner_tasks.cpp @@ -7,7 +7,6 @@ #include "inner_tasks.h" #include "scheduler.h" -#include "../GlobalTaskGroup.h" #include "../../StockManager.h" namespace hku { diff --git a/hikyuu_cpp/hikyuu/global/schedule/scheduler.cpp b/hikyuu_cpp/hikyuu/global/schedule/scheduler.cpp index 90d58f4e..c2deaa72 100644 --- a/hikyuu_cpp/hikyuu/global/schedule/scheduler.cpp +++ b/hikyuu_cpp/hikyuu/global/schedule/scheduler.cpp @@ -8,7 +8,6 @@ #include "hikyuu/GlobalInitializer.h" #include #include "hikyuu/utilities/Log.h" -#include "hikyuu/global/GlobalTaskGroup.h" #include "scheduler.h" namespace hku { @@ -17,8 +16,7 @@ static TimerManager *g_scheduler; TimerManager *getScheduler() { static std::once_flag oc; - // 使用内部公共任务组，减少内部线程 - std::call_once(oc, [&]() { g_scheduler = new TimerManager(getGlobalTaskGroup()); }); + std::call_once(oc, [&]() { g_scheduler = new TimerManager(1); }); return g_scheduler; } diff --git a/hikyuu_cpp/hikyuu/hikyuu.cpp b/hikyuu_cpp/hikyuu/hikyuu.cpp index 478c9776..1654777f 100644 --- a/hikyuu_cpp/hikyuu/hikyuu.cpp +++ b/hikyuu_cpp/hikyuu/hikyuu.cpp @@ -53,14 +53,6 @@ void HKU_API getConfigFromIni(const string& config_file_name, Parameter& basePar hkuParam.set("load_history_finance", config.getBool("hikyuu", "load_history_finance", "True")); - // 公共线程池线程数量 - size_t cpu_num = 2 * std::thread::hardware_concurrency(); - if (cpu_num > 30) { - cpu_num = 30; - } - hkuParam.set("commont_thread_pool_num", config.getInt("hikyuu", "commont_thread_pool_num", - fmt::format("{}", cpu_num))); - IniParser::StringListPtr option = config.getOptionList("baseinfo"); for (auto iter = option->begin(); iter != option->end(); ++iter) { string value = config.get("baseinfo", *iter); diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp index e6967c79..b937ca4f 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp @@ -12,19 +12,10 @@ #include "hikyuu/utilities/node/NodeClient.h" #include "hikyuu/global/GlobalSpotAgent.h" #include "hikyuu/global/schedule/scheduler.h" -#include "hikyuu/global/GlobalTaskGroup.h" #include "hikyuu/global/sysinfo.h" #include "hikyuu/hikyuu.h" #include "Strategy.h" -// python 中运行拉回主线程循环，非 python 环境则直接执行 -#define EVENT(func) \ - if (runningInPython()) { \ - event(func); \ - } else { \ - func(); \ - } - namespace hku { std::atomic_bool Strategy::ms_keep_running = true; @@ -111,7 +102,7 @@ void Strategy::start() { agent.addProcess([this](const SpotRecord& spot) { _receivedSpot(spot); }); agent.addPostProcess([this](Datetime revTime) { if (m_on_recieved_spot) { - EVENT([=]() { m_on_recieved_spot(revTime); }); + event([=]() { m_on_recieved_spot(revTime); }); } }); startSpotAgent(true); @@ -136,7 +127,7 @@ void Strategy::_receivedSpot(const SpotRecord& spot) { Stock stk = getStock(format("{}{}", spot.market, spot.code)); if (!stk.isNull()) { if (m_on_change) { - EVENT([=]() { m_on_change(stk, spot); }); + event([=]() { m_on_change(stk, spot); }); } } } @@ -149,7 +140,7 @@ void Strategy::runDaily(std::function&& func, const TimeDelta& delta, m_ignoreMarket = ignoreMarket; if (ignoreMarket) { - m_run_daily_func = [=]() { EVENT(func); }; + m_run_daily_func = [=]() { event(func); }; } else { m_run_daily_func = [=]() { @@ -167,7 +158,7 @@ void Strategy::runDaily(std::function&& func, const TimeDelta& delta, Datetime close2 = today + market_info.closeTime2(); Datetime now = Datetime::now(); if ((now >= open1 && now <= close1) || (now >= open2 && now <= close2)) { - EVENT(func); + event(func); } }; } @@ -264,12 +255,12 @@ void Strategy::runDailyAt(std::function&& func, const TimeDelta& delta, auto today = Datetime::today(); int day = today.dayOfWeek(); if (day != 0 && day != 6 && !sm.isHoliday(today)) { - EVENT(func); + event(func); } }; } else { - m_run_daily_at_func = [=]() { EVENT(func); }; + m_run_daily_at_func = [=]() { event(func); }; } } From d360deb079ffb7edb7c850b6744d67a282255df9 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Fri, 30 Aug 2024 23:09:07 +0800 Subject: [PATCH 15/21] fixed Operator SE serialization leak --- .../selector/imp/OperatorSelector.cpp | 20 +++++++++++-------- .../trade_sys/selector/imp/OperatorSelector.h | 11 +++++++++- .../selector/imp/OperatorValueSelector.h | 19 +++++++++++++++++- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.cpp b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.cpp index ed87da2c..1cbdf948 100644 --- a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.cpp +++ b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.cpp @@ -50,10 +50,16 @@ OperatorSelector::OperatorSelector(const string& name) : SelectorBase(name) {} OperatorSelector::OperatorSelector(const string& name, const SelectorPtr& se1, const SelectorPtr& se2) : SelectorBase(name), m_se1(se1), m_se2(se2) { - auto inter = findIntersection(se1, se2); - if (se1 && se2) { + build(); +} + +OperatorSelector::~OperatorSelector() {} + +void OperatorSelector::build() { + auto inter = findIntersection(m_se1, m_se2); + if (m_se1 && m_se2) { std::map tmpdict; - const auto& raw_sys_list1 = se1->getProtoSystemList(); + const auto& raw_sys_list1 = m_se1->getProtoSystemList(); for (const auto& sys : raw_sys_list1) { m_pro_sys_list.emplace_back(sys); m_se1_set.insert(sys); @@ -62,7 +68,7 @@ OperatorSelector::OperatorSelector(const string& name, const SelectorPtr& se1, } } - const auto& raw_sys_list2 = se2->getProtoSystemList(); + const auto& raw_sys_list2 = m_se2->getProtoSystemList(); for (size_t i = 0, total = raw_sys_list2.size(); i < total; i++) { const auto& sys = raw_sys_list2[i]; auto iter = inter.find(sys.get()); @@ -74,7 +80,7 @@ OperatorSelector::OperatorSelector(const string& name, const SelectorPtr& se1, } } - } else if (se1) { + } else if (m_se1) { // m_se1 = se1->clone(); auto sys_list = m_se1->getProtoSystemList(); for (auto& sys : sys_list) { @@ -82,7 +88,7 @@ OperatorSelector::OperatorSelector(const string& name, const SelectorPtr& se1, } m_pro_sys_list = std::move(sys_list); - } else if (se2) { + } else if (m_se2) { // m_se2 = se2->clone(); auto sys_list = m_se2->getProtoSystemList(); for (auto& sys : sys_list) { @@ -92,8 +98,6 @@ OperatorSelector::OperatorSelector(const string& name, const SelectorPtr& se1, } } -OperatorSelector::~OperatorSelector() {} - void OperatorSelector::_reset() { if (m_se1) { m_se1->reset(); diff --git a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.h b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.h index 2a52aab0..471a6e95 100644 --- a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.h +++ b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorSelector.h @@ -43,6 +43,7 @@ protected: protected: static void sortSystemWeightList(SystemWeightList& swlist); + void build(); void cloneRebuild(const SelectorPtr& se1, const SelectorPtr& se2); protected: @@ -63,11 +64,19 @@ private: #if HKU_SUPPORT_SERIALIZATION friend class boost::serialization::access; template - void serialize(Archive& ar, const unsigned int version) { + void save(Archive& ar, const unsigned int version) const { ar& BOOST_SERIALIZATION_BASE_OBJECT_NVP(SelectorBase); ar& BOOST_SERIALIZATION_NVP(m_se1); ar& BOOST_SERIALIZATION_NVP(m_se2); } + + template + void load(Archive& ar, const unsigned int version) { + ar& BOOST_SERIALIZATION_BASE_OBJECT_NVP(SelectorBase); + ar& BOOST_SERIALIZATION_NVP(m_se1); + ar& BOOST_SERIALIZATION_NVP(m_se2); + build(); + } #endif }; diff --git a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorValueSelector.h b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorValueSelector.h index 004213cb..f85d171a 100644 --- a/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorValueSelector.h +++ b/hikyuu_cpp/hikyuu/trade_sys/selector/imp/OperatorValueSelector.h @@ -36,12 +36,29 @@ private: //============================================ #if HKU_SUPPORT_SERIALIZATION friend class boost::serialization::access; + // template + // void serialize(Archive& ar, const unsigned int version) { + // ar& BOOST_SERIALIZATION_BASE_OBJECT_NVP(SelectorBase); + // ar& BOOST_SERIALIZATION_NVP(m_se); + // ar& BOOST_SERIALIZATION_NVP(m_value); + // } template - void serialize(Archive& ar, const unsigned int version) { + void save(Archive& ar, const unsigned int version) const { ar& BOOST_SERIALIZATION_BASE_OBJECT_NVP(SelectorBase); ar& BOOST_SERIALIZATION_NVP(m_se); ar& BOOST_SERIALIZATION_NVP(m_value); } + + template + void load(Archive& ar, const unsigned int version) { + ar& BOOST_SERIALIZATION_BASE_OBJECT_NVP(SelectorBase); + ar& BOOST_SERIALIZATION_NVP(m_se); + ar& BOOST_SERIALIZATION_NVP(m_value); + if (m_se) { + m_pro_sys_list = m_se->getProtoSystemList(); + } + } + #endif }; From 9038473eb8dba8d99440b1c4f5778eab9c1b5d0f Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 01:31:25 +0800 Subject: [PATCH 16/21] =?UTF-8?q?fixed=20=E6=96=B0=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E5=8A=A0=E8=BD=BDlinux=E4=B8=8B=E9=80=80=E5=87=BA=E6=97=B6hdf5?= =?UTF-8?q?=E5=85=B3=E9=97=AD=E5=BC=82=E5=B8=B8=EF=BC=9B=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E5=B0=8F=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/GlobalInitializer.cpp | 8 +++++ hikyuu_cpp/hikyuu/StockManager.cpp | 34 +++++++++++++------- hikyuu_cpp/hikyuu/StockManager.h | 12 +++++-- hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp | 10 ++++-- hikyuu_cpp/hikyuu/global/agent/SpotAgent.h | 5 +-- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp index 5da311f9..a01a3905 100644 --- a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp +++ b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp @@ -89,6 +89,14 @@ void GlobalInitializer::clean() { IndicatorImp::releaseDynEngine(); +#if !HKU_OS_WINDOWS + // 主动停止异步数据加载任务组，否则 hdf5 在 linux 下会报关闭异常 + auto *tg = StockManager::instance().getLoadTaskGroup(); + if (tg) { + tg->stop(); + } +#endif + #if HKU_ENABLE_LEAK_DETECT || defined(MSVC_LEAKER_DETECT) // 非内存泄漏检测时，内存让系统自动释放，避免某些场景下 windows 下退出速度过慢 StockManager::quit(); diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 822e5513..d97f0b63 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -105,16 +105,23 @@ void StockManager::loadData() { std::chrono::system_clock::time_point start_time = std::chrono::system_clock::now(); m_data_ready = false; - loadAllHolidays(); + ThreadPool tg(2); + tg.submit([this]() { this->loadAllHolidays(); }); + tg.submit([this]() { this->loadHistoryFinanceField(); }); + tg.submit([this]() { this->loadAllStockTypeInfo(); }); + tg.submit([this]() { this->loadAllZhBond10(); }); + + // loadAllHolidays(); loadAllMarketInfos(); - loadAllStockTypeInfo(); + // loadAllStockTypeInfo(); loadAllStocks(); loadAllStockWeights(); - loadAllZhBond10(); - loadHistoryFinanceField(); + // loadAllZhBond10(); + // loadHistoryFinanceField(); HKU_INFO("Loading block..."); - m_blockDriver->load(); + // m_blockDriver->load(); + tg.submit([this]() { this->m_blockDriver->load(); }); // 获取K线数据驱动并预加载指定的数据 HKU_INFO("Loading KData..."); @@ -124,6 +131,8 @@ void StockManager::loadData() { // 加载K线及历史财务信息 loadAllKData(); + tg.join(); + std::chrono::duration sec = std::chrono::system_clock::now() - start_time; HKU_INFO("{:<.2f}s Loaded Data.", sec.count()); } @@ -171,14 +180,16 @@ void StockManager::loadAllKData() { } else { // 异步并行加载 std::thread t = std::thread([this, ktypes, low_ktypes]() { - ThreadPool tg(std::thread::hardware_concurrency()); + this->m_load_tg = std::make_unique(); + // ThreadPool tg(std::thread::hardware_concurrency()); for (size_t i = 0, len = ktypes.size(); i < len; i++) { std::lock_guard lock(*m_stockDict_mutex); for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { if (m_preloadParam.tryGet(low_ktypes[i], false)) { - tg.submit([stk = iter->second, ktype = std::move(ktypes[i])]() mutable { - stk.loadKDataToBuffer(ktype); - }); + m_load_tg->submit( + [stk = iter->second, ktype = std::move(ktypes[i])]() mutable { + stk.loadKDataToBuffer(ktype); + }); } } } @@ -186,11 +197,12 @@ void StockManager::loadAllKData() { if (m_hikyuuParam.tryGet("load_history_finance", true)) { std::lock_guard lock(*m_stockDict_mutex); for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { - tg.submit([stk = iter->second]() { stk.getHistoryFinance(); }); + m_load_tg->submit([stk = iter->second]() { stk.getHistoryFinance(); }); } } - tg.join(); + m_load_tg->join(); + m_load_tg.reset(); m_data_ready = true; }); t.detach(); diff --git a/hikyuu_cpp/hikyuu/StockManager.h b/hikyuu_cpp/hikyuu/StockManager.h index 33a78103..38b7136b 100644 --- a/hikyuu_cpp/hikyuu/StockManager.h +++ b/hikyuu_cpp/hikyuu/StockManager.h @@ -10,8 +10,9 @@ #include #include -#include "utilities/Parameter.h" -#include "data_driver/DataDriverFactory.h" +#include "hikyuu/utilities/Parameter.h" +#include "hikyuu/utilities/thread/thread.h" +#include "hikyuu/data_driver/DataDriverFactory.h" #include "Block.h" #include "MarketInfo.h" #include "StockTypeInfo.h" @@ -218,6 +219,11 @@ public: return m_thread_id; } + /** 仅由程序退出使使用！！！ */ + ThreadPool* getLoadTaskGroup() { + return m_load_tg.get(); + } + public: typedef StockMapIterator const_iterator; const_iterator begin() const { @@ -293,6 +299,8 @@ private: Parameter m_preloadParam; Parameter m_hikyuuParam; StrategyContext m_context; + + std::unique_ptr m_load_tg; // 异步数据加载辅助线程组 }; inline size_t StockManager::size() const { diff --git a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp index 7c2eb084..818a757a 100644 --- a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp @@ -25,14 +25,18 @@ const size_t SpotAgent::ms_endTagLength = strlen(SpotAgent::ms_endTag); Datetime SpotAgent::ms_start_rev_time; -void SpotAgent::setQuotationServer(const string& server) { - ms_pubUrl = server; +SpotAgent::SpotAgent() { + m_tg = std::make_unique(); } SpotAgent::~SpotAgent() { stop(); } +void SpotAgent::setQuotationServer(const string& server) { + ms_pubUrl = server; +} + void SpotAgent::start() { stop(); if (m_stop) { @@ -138,7 +142,7 @@ void SpotAgent::parseSpotData(const void* buf, size_t buf_len) { auto spot_record = parseFlatSpot(spot); if (spot_record) { for (const auto& process : m_processList) { - m_process_task_list.emplace_back(m_tg.submit(ProcessTask(process, *spot_record))); + m_process_task_list.emplace_back(m_tg->submit(ProcessTask(process, *spot_record))); } } } diff --git a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h index cafc4b4e..aaa703b3 100644 --- a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h +++ b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h @@ -27,7 +27,7 @@ namespace hku { */ class HKU_API SpotAgent { public: - SpotAgent() = default; + SpotAgent(); /** 析构函数 */ virtual ~SpotAgent(); @@ -109,7 +109,8 @@ private: int m_revTimeout = 100; // 连接数据服务超时时长（毫秒） size_t m_batch_count = 0; // 记录本次批次接收的数据数量 std::thread m_receiveThread; // 数据接收线程 - ThreadPool m_tg; // 数据处理任务线程池 + // ThreadPool m_tg; // 数据处理任务线程池 + std::unique_ptr m_tg; vector> m_process_task_list; // 下面属性被修改时需要加锁，以便可以使用多线程方式运行 strategy From d8d82008c3b0d48e3a3de2556dcbac32040cc337 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 02:26:27 +0800 Subject: [PATCH 17/21] update --- hikyuu_cpp/hikyuu/GlobalInitializer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp index a01a3905..632a31bc 100644 --- a/hikyuu_cpp/hikyuu/GlobalInitializer.cpp +++ b/hikyuu_cpp/hikyuu/GlobalInitializer.cpp @@ -89,13 +89,11 @@ void GlobalInitializer::clean() { IndicatorImp::releaseDynEngine(); -#if !HKU_OS_WINDOWS // 主动停止异步数据加载任务组，否则 hdf5 在 linux 下会报关闭异常 auto *tg = StockManager::instance().getLoadTaskGroup(); if (tg) { tg->stop(); } -#endif #if HKU_ENABLE_LEAK_DETECT || defined(MSVC_LEAKER_DETECT) // 非内存泄漏检测时，内存让系统自动释放，避免某些场景下 windows 下退出速度过慢 From 8fe41fca91c6095040cf3edd50fec510b0447f31 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 03:00:06 +0800 Subject: [PATCH 18/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=20Strategy=20=E8=A1=8C?= =?UTF-8?q?=E6=83=85=E6=95=B0=E6=8D=AE=E5=A4=84=E4=BB=BB=E5=8A=A1=E7=BB=84?= =?UTF-8?q?=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/StockManager.cpp | 1 - hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp | 8 ++++---- hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h | 5 +++-- hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp | 4 ++-- hikyuu_cpp/hikyuu/global/agent/SpotAgent.h | 2 +- hikyuu_cpp/hikyuu/strategy/Strategy.cpp | 9 ++++++++- hikyuu_pywrap/global/_SpotAgent.cpp | 3 ++- 7 files changed, 20 insertions(+), 12 deletions(-) diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index d97f0b63..7729f25b 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -181,7 +181,6 @@ void StockManager::loadAllKData() { // 异步并行加载 std::thread t = std::thread([this, ktypes, low_ktypes]() { this->m_load_tg = std::make_unique(); - // ThreadPool tg(std::thread::hardware_concurrency()); for (size_t i = 0, len = ktypes.size(); i < len; i++) { std::lock_guard lock(*m_stockDict_mutex); for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) { diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp index 80b37c0f..23d3e619 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.cpp @@ -14,9 +14,9 @@ namespace hku { SpotAgent* g_spot_agent = nullptr; -SpotAgent* getGlobalSpotAgent() { +SpotAgent* getGlobalSpotAgent(size_t worker_num) { if (!g_spot_agent) { - g_spot_agent = new SpotAgent(); + g_spot_agent = new SpotAgent(worker_num); } return g_spot_agent; } @@ -169,11 +169,11 @@ static void updateStockMinData(const SpotRecord& spot, KQuery::KType ktype) { stk.realtimeUpdate(krecord, ktype); } -void HKU_API startSpotAgent(bool print) { +void HKU_API startSpotAgent(bool print, size_t worker_num) { StockManager& sm = StockManager::instance(); SpotAgent::setQuotationServer( sm.getHikyuuParameter().tryGet("quotation_server", "ipc:///tmp/hikyuu_real.ipc")); - auto& agent = *getGlobalSpotAgent(); + auto& agent = *getGlobalSpotAgent(worker_num); HKU_CHECK(!agent.isRunning(), "The agent is running, please stop first!"); agent.setPrintFlag(print); diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h index 5df25ed9..077822b6 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h @@ -15,7 +15,8 @@ namespace hku { * @param print 打印接收数据进展 * @ingroup Agent */ -void HKU_API startSpotAgent(bool print = true); +void HKU_API startSpotAgent(bool print = true, + size_t worker_num = std::thread::hardware_concurrency()); /** * 终止 Spot 数据接收代理 @@ -23,7 +24,7 @@ void HKU_API startSpotAgent(bool print = true); */ void HKU_API stopSpotAgent(); -SpotAgent* getGlobalSpotAgent(); +SpotAgent* getGlobalSpotAgent(size_t worker_num = std::thread::hardware_concurrency()); void releaseGlobalSpotAgent(); diff --git a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp index 818a757a..37edc342 100644 --- a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp +++ b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.cpp @@ -25,8 +25,8 @@ const size_t SpotAgent::ms_endTagLength = strlen(SpotAgent::ms_endTag); Datetime SpotAgent::ms_start_rev_time; -SpotAgent::SpotAgent() { - m_tg = std::make_unique(); +SpotAgent::SpotAgent(size_t worker_num) { + m_tg = std::make_unique(worker_num); } SpotAgent::~SpotAgent() { diff --git a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h index aaa703b3..f232172e 100644 --- a/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h +++ b/hikyuu_cpp/hikyuu/global/agent/SpotAgent.h @@ -27,7 +27,7 @@ namespace hku { */ class HKU_API SpotAgent { public: - SpotAgent(); + SpotAgent(size_t worker_num); /** 析构函数 */ virtual ~SpotAgent(); diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp index b937ca4f..775c06d1 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp @@ -98,7 +98,14 @@ void Strategy::start() { _runDailyAt(); - auto& agent = *getGlobalSpotAgent(); + size_t stock_num = StockManager::instance().size(); + size_t spot_worker_num = stock_num / 300; + size_t cpu_num = std::thread::hardware_concurrency(); + if (spot_worker_num > cpu_num) { + spot_worker_num = cpu_num; + } + + auto& agent = *getGlobalSpotAgent(spot_worker_num); agent.addProcess([this](const SpotRecord& spot) { _receivedSpot(spot); }); agent.addPostProcess([this](Datetime revTime) { if (m_on_recieved_spot) { diff --git a/hikyuu_pywrap/global/_SpotAgent.cpp b/hikyuu_pywrap/global/_SpotAgent.cpp index 71ecfd1d..f69ae2bc 100644 --- a/hikyuu_pywrap/global/_SpotAgent.cpp +++ b/hikyuu_pywrap/global/_SpotAgent.cpp @@ -12,6 +12,7 @@ using namespace hku; namespace py = pybind11; void export_SpotAgent(py::module& m) { - m.def("start_spot_agent", startSpotAgent, py::arg("print") = false); + m.def("start_spot_agent", startSpotAgent, py::arg("print") = false, + py::arg("worker_num") = std::thread::hardware_concurrency()); m.def("stop_spot_agent", stopSpotAgent); } From 6995629d409b619fa8420af43addb8e621562b75 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 03:14:46 +0800 Subject: [PATCH 19/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=20Strategy=20start=20?= =?UTF-8?q?=E8=87=AA=E8=A1=8C=E6=8E=A7=E5=88=B6=E6=98=AF=E5=90=A6=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E8=A1=8C=E6=83=85=E6=8E=A5=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu_cpp/hikyuu/strategy/Strategy.cpp | 32 +++++++++++++------------ hikyuu_cpp/hikyuu/strategy/Strategy.h | 2 +- hikyuu_pywrap/strategy/_Strategy.cpp | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp index 775c06d1..f2824b73 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.cpp +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.cpp @@ -93,26 +93,28 @@ void Strategy::_init() { stopSpotAgent(); } -void Strategy::start() { +void Strategy::start(bool autoRecieveSpot) { _init(); _runDailyAt(); - size_t stock_num = StockManager::instance().size(); - size_t spot_worker_num = stock_num / 300; - size_t cpu_num = std::thread::hardware_concurrency(); - if (spot_worker_num > cpu_num) { - spot_worker_num = cpu_num; - } - - auto& agent = *getGlobalSpotAgent(spot_worker_num); - agent.addProcess([this](const SpotRecord& spot) { _receivedSpot(spot); }); - agent.addPostProcess([this](Datetime revTime) { - if (m_on_recieved_spot) { - event([=]() { m_on_recieved_spot(revTime); }); + if (autoRecieveSpot) { + size_t stock_num = StockManager::instance().size(); + size_t spot_worker_num = stock_num / 300; + size_t cpu_num = std::thread::hardware_concurrency(); + if (spot_worker_num > cpu_num) { + spot_worker_num = cpu_num; } - }); - startSpotAgent(true); + + auto& agent = *getGlobalSpotAgent(spot_worker_num); + agent.addProcess([this](const SpotRecord& spot) { _receivedSpot(spot); }); + agent.addPostProcess([this](Datetime revTime) { + if (m_on_recieved_spot) { + event([=]() { m_on_recieved_spot(revTime); }); + } + }); + startSpotAgent(true); + } _runDaily(); diff --git a/hikyuu_cpp/hikyuu/strategy/Strategy.h b/hikyuu_cpp/hikyuu/strategy/Strategy.h index e59fa451..f461a11a 100644 --- a/hikyuu_cpp/hikyuu/strategy/Strategy.h +++ b/hikyuu_cpp/hikyuu/strategy/Strategy.h @@ -89,7 +89,7 @@ public: /** * 启动策略执行，必须在已注册相关处理函数后执行 */ - void start(); + void start(bool autoRecieveSpot = true); private: string m_name; diff --git a/hikyuu_pywrap/strategy/_Strategy.cpp b/hikyuu_pywrap/strategy/_Strategy.cpp index 4309b1bd..8b4f1c8f 100644 --- a/hikyuu_pywrap/strategy/_Strategy.cpp +++ b/hikyuu_pywrap/strategy/_Strategy.cpp @@ -30,7 +30,7 @@ void export_Strategy(py::module& m) { .def_property_readonly("context", &Strategy::context, py::return_value_policy::copy, "策略上下文") - .def("start", &Strategy::start) + .def("start", &Strategy::start, py::arg("auto_recieve_spot") = true) .def("on_change", [](Strategy& self, py::object func) { HKU_CHECK(py::hasattr(func, "__call__"), "func is not callable!"); From b8333c7e59076a17eb8f11594630e589e4ca6c25 Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 10:12:42 +0800 Subject: [PATCH 20/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8A=A0=E8=BD=BD=20hi?= =?UTF-8?q?kyuu.interactive=20=E5=8F=AF=E9=80=9A=E8=BF=87=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E5=8F=98=E9=87=8F=E4=B8=8E=E4=B8=8A=E4=B8=8B=E6=96=87?= =?UTF-8?q?=E6=8E=A7=E5=88=B6=E7=9B=B8=E5=85=B3=E6=95=B0=E6=8D=AE=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hikyuu/interactive.py | 16 +++++++++++- hikyuu_cpp/hikyuu/StockManager.cpp | 26 +++++++++++++------ hikyuu_cpp/hikyuu/StrategyContext.cpp | 14 +++++----- hikyuu_cpp/hikyuu/StrategyContext.h | 9 +++++-- hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h | 2 +- .../hikyuu/hikyuu/test_StrategyContext.cpp | 4 +-- hikyuu_pywrap/_StockManager.cpp | 3 +++ hikyuu_pywrap/_StrategyContext.cpp | 17 ++++++++---- 8 files changed, 64 insertions(+), 27 deletions(-) diff --git a/hikyuu/interactive.py b/hikyuu/interactive.py index 8bc0ee6b..26633c45 100644 --- a/hikyuu/interactive.py +++ b/hikyuu/interactive.py @@ -112,7 +112,21 @@ for p in kdata_config: continue kdata_param[p] = ini.get('kdata', p) -sm.init(base_param, block_param, kdata_param, preload_param, hku_param) +context = StrategyContext() +if 'HKU_STOCK_LIST' in os.environ: + context.stock_list = os.environ['HKU_STOCK_LIST'].split(";") +if 'HKU_KTYPE_LIST' in os.environ: + context.ktype_list = os.environ['HKU_KTYPE_LIST'].split(";") +if 'HKU_LOAD_HISTORY_FINANCE' in os.environ: + load_str = os.environ['HKU_LOAD_HISTORY_FINANCE'].upper() + load_finance = os.environ['HKU_LOAD_HISTORY_FINANCE'] in ("1", "TRUE") + hku_param.set("load_history_finance", load_finance) +if 'HKU_LOAD_STOCK_WEIGHT' in os.environ: + load_str = os.environ['HKU_LOAD_STOCK_WEIGHT'].upper() + load_stk_weight = os.environ['HKU_LOAD_STOCK_WEIGHT'] in ("1", "TRUE") + hku_param.set("load_stock_weight", load_stk_weight) + +sm.init(base_param, block_param, kdata_param, preload_param, hku_param, context) # set_log_level(LOG_LEVEL.INFO) # 启动行情接收代理 diff --git a/hikyuu_cpp/hikyuu/StockManager.cpp b/hikyuu_cpp/hikyuu/StockManager.cpp index 7729f25b..8a96acd7 100644 --- a/hikyuu_cpp/hikyuu/StockManager.cpp +++ b/hikyuu_cpp/hikyuu/StockManager.cpp @@ -139,14 +139,24 @@ void StockManager::loadData() { void StockManager::loadAllKData() { // 按 K 线类型控制加载顺序 - // const auto& ktypes = KQuery::getAllKType(); - vector ktypes{KQuery::DAY, KQuery::MIN, KQuery::WEEK, KQuery::MONTH, - KQuery::QUARTER, KQuery::HALFYEAR, KQuery::YEAR, KQuery::MIN5, - KQuery::MIN15, KQuery::MIN30, KQuery::MIN60, KQuery::MIN3, - KQuery::HOUR2, KQuery::HOUR4, KQuery::HOUR6, KQuery::HOUR12}; - HKU_ASSERT(ktypes.size() == KQuery::getAllKType().size()); + vector default_ktypes{ + KQuery::DAY, KQuery::MIN, KQuery::WEEK, KQuery::MONTH, KQuery::QUARTER, KQuery::HALFYEAR, + KQuery::YEAR, KQuery::MIN5, KQuery::MIN15, KQuery::MIN30, KQuery::MIN60, KQuery::MIN3, + KQuery::HOUR2, KQuery::HOUR4, KQuery::HOUR6, KQuery::HOUR12}; + vector ktypes; vector low_ktypes; + + // 如果上下文指定了 ktype list，则按上下文指定的 ktype 顺序加载，否则按默认顺序加载 + const auto& context_ktypes = m_context.getKTypeList(); + if (context_ktypes.empty()) { + ktypes = std::move(default_ktypes); + HKU_ASSERT(ktypes.size() == KQuery::getAllKType().size()); + + } else { + ktypes = context_ktypes; + } + low_ktypes.reserve(ktypes.size()); for (const auto& ktype : ktypes) { auto& back = low_ktypes.emplace_back(ktype); @@ -373,9 +383,9 @@ void StockManager::loadAllStocks() { if (m_context.isAll()) { stockInfos = m_baseInfoDriver->getAllStockInfo(); } else { - const vector& context_stock_code_list = m_context.getStockCodeList(); + auto load_stock_code_list = m_context.getAllNeedLoadStockCodeList(); auto all_market = getAllMarket(); - for (auto stkcode : context_stock_code_list) { + for (auto stkcode : load_stock_code_list) { to_upper(stkcode); bool find = false; for (auto& market : all_market) { diff --git a/hikyuu_cpp/hikyuu/StrategyContext.cpp b/hikyuu_cpp/hikyuu/StrategyContext.cpp index 6a730f8e..52eb60fe 100644 --- a/hikyuu_cpp/hikyuu/StrategyContext.cpp +++ b/hikyuu_cpp/hikyuu/StrategyContext.cpp @@ -21,12 +21,6 @@ void StrategyContext::setKTypeList(const vector& ktypeList) { to_upper(ktype); return ktype; }); - - // 对 ktype 按时间长度进行升序排序 - std::sort(m_ktypeList.begin(), m_ktypeList.end(), - [](const KQuery::KType& a, const KQuery::KType& b) { - return KQuery::getKTypeInMin(a) < KQuery::getKTypeInMin(b); - }); } bool StrategyContext::isAll() const noexcept { @@ -36,8 +30,12 @@ bool StrategyContext::isAll() const noexcept { }) != m_stockCodeList.end(); } -bool StrategyContext::isValid() const noexcept { - return m_stockCodeList.empty() || m_ktypeList.empty(); +vector StrategyContext::getAllNeedLoadStockCodeList() const { + vector ret{m_stockCodeList}; + for (const auto& code : m_mustLoad) { + ret.push_back(code); + } + return ret; } } // namespace hku \ No newline at end of file diff --git a/hikyuu_cpp/hikyuu/StrategyContext.h b/hikyuu_cpp/hikyuu/StrategyContext.h index 133f43b2..70f48aec 100644 --- a/hikyuu_cpp/hikyuu/StrategyContext.h +++ b/hikyuu_cpp/hikyuu/StrategyContext.h @@ -29,8 +29,6 @@ public: bool isAll() const noexcept; - bool isValid() const noexcept; - Datetime startDatetime() const noexcept { return m_startDatetime; } @@ -57,8 +55,15 @@ public: return m_ktypeList; } + const vector& getMustLoadStockCodeList() const { + return m_mustLoad; + } + + vector getAllNeedLoadStockCodeList() const; + private: Datetime m_startDatetime{19901219}; + vector m_mustLoad{"sh000001", "sh000300"}; // 默认必须加载的 stock vector m_stockCodeList; vector m_ktypeList; }; diff --git a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h index 077822b6..33b52c66 100644 --- a/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h +++ b/hikyuu_cpp/hikyuu/global/GlobalSpotAgent.h @@ -26,6 +26,6 @@ void HKU_API stopSpotAgent(); SpotAgent* getGlobalSpotAgent(size_t worker_num = std::thread::hardware_concurrency()); -void releaseGlobalSpotAgent(); +void HKU_API releaseGlobalSpotAgent(); } // namespace hku \ No newline at end of file diff --git a/hikyuu_cpp/unit_test/hikyuu/hikyuu/test_StrategyContext.cpp b/hikyuu_cpp/unit_test/hikyuu/hikyuu/test_StrategyContext.cpp index b36da36c..bd9f83c9 100644 --- a/hikyuu_cpp/unit_test/hikyuu/hikyuu/test_StrategyContext.cpp +++ b/hikyuu_cpp/unit_test/hikyuu/hikyuu/test_StrategyContext.cpp @@ -20,8 +20,8 @@ TEST_CASE("test_StrategyContext") { sc.setKTypeList( {KQuery::MONTH, KQuery::MIN5, KQuery::DAY, KQuery::MIN, KQuery::WEEK, KQuery::MIN60}); - vector expect{KQuery::MIN, KQuery::MIN5, KQuery::MIN60, - KQuery::DAY, KQuery::WEEK, KQuery::MONTH}; + vector expect{KQuery::MONTH, KQuery::MIN5, KQuery::DAY, + KQuery::MIN, KQuery::WEEK, KQuery::MIN60}; const auto ktypes = sc.getKTypeList(); for (size_t i = 0, len = ktypes.size(); i < len; i++) { CHECK_EQ(ktypes[i], expect[i]); diff --git a/hikyuu_pywrap/_StockManager.cpp b/hikyuu_pywrap/_StockManager.cpp index 9d2f05e6..65b88e22 100644 --- a/hikyuu_pywrap/_StockManager.cpp +++ b/hikyuu_pywrap/_StockManager.cpp @@ -31,6 +31,9 @@ void export_StockManager(py::module& m) { param hikyuu_param 其他参数 param StrategyContext context 策略上下文, 默认加载全部证券)") + .def_property_readonly("data_ready", &StockManager::dataReady, + "是否所有数据已准备就绪（加载完毕）") + .def("reload", &StockManager::reload, "重新加载所有证券数据") .def("tmpdir", &StockManager::tmpdir, R"(tmpdir(self) -> str diff --git a/hikyuu_pywrap/_StrategyContext.cpp b/hikyuu_pywrap/_StrategyContext.cpp index 4f93c4ae..29dfa482 100644 --- a/hikyuu_pywrap/_StrategyContext.cpp +++ b/hikyuu_pywrap/_StrategyContext.cpp @@ -32,9 +32,16 @@ void export_StrategeContext(py::module& m) { .def(py::init&, const vector&>(), py::arg("stock_list"), py::arg("ktype_list")) .def_property("start_datetime", get_start_datetime, set_start_datetime, "起始日期") - .def_property("stock_list", - py::overload_cast<>(&StrategyContext::getStockCodeList, py::const_), - setStockList, "股票代码列表") - .def_property("ktype_list", py::overload_cast<>(&StrategyContext::getKTypeList, py::const_), - setKTypeList, "需要的K线类型"); + .def_property( + "stock_list", py::overload_cast<>(&StrategyContext::getStockCodeList, py::const_), + [](StrategyContext& self, const py::sequence& stk_list) { + self.setStockCodeList(python_list_to_vector(stk_list)); + }, + "股票代码列表") + .def_property( + "ktype_list", py::overload_cast<>(&StrategyContext::getKTypeList, py::const_), + [](StrategyContext& self, const py::sequence& ktype_list) { + self.setKTypeList(python_list_to_vector(ktype_list)); + }, + "需要的K线类型"); } From 0a0fe2ed9aeb0f042a23e9e7711b45741b9b6e4c Mon Sep 17 00:00:00 2001 From: fasiondog Date: Sat, 31 Aug 2024 10:15:00 +0800 Subject: [PATCH 21/21] update --- hikyuu_cpp/demo/demo3.cpp | 116 -------------------------------------- hikyuu_cpp/demo/xmake.lua | 24 -------- 2 files changed, 140 deletions(-) delete mode 100644 hikyuu_cpp/demo/demo3.cpp diff --git a/hikyuu_cpp/demo/demo3.cpp b/hikyuu_cpp/demo/demo3.cpp deleted file mode 100644 index dd270185..00000000 --- a/hikyuu_cpp/demo/demo3.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// demo.cpp : 定义控制台应用程序的入口点。 -// - -#include -#include -#include -#include -#include -#include -#include - -#if defined(_WIN32) -#include -#endif - -using namespace hku; - -NodeServer server; - -void signal_handle(int signal) { - if (signal == SIGINT || signal == SIGTERM) { - HKU_INFO("Shutdown now ..."); - server.stop(); - exit(0); - } -} - -int main(int argc, char* argv[]) { - std::signal(SIGINT, signal_handle); - std::signal(SIGTERM, signal_handle); - - initLogger(false, "./demo.log"); - -#if defined(_WIN32) - // Windows 下设置控制台程序输出代码页为 UTF8 - auto old_cp = GetConsoleOutputCP(); - SetConsoleOutputCP(CP_UTF8); -#endif - - try { - // 配置文件的位置自行修改 - hikyuu_init(fmt::format("{}/.hikyuu/hikyuu.ini", getUserDir())); - - // 启动行情接收（只是计算回测可以不需要） - startSpotAgent(true); - - server.setAddr("tcp://0.0.0.0:9201"); - - server.regHandle("market", [](json&& req) { - HKU_INFO("--> req from {}:{}", req["remote_host"].get(), - req["remote_port"].get()); - HKU_ASSERT(req.contains("ktype")); - HKU_ASSERT(req.contains("codes")); - - string ktype = req["ktype"].get(); - auto& sm = StockManager::instance(); - const auto& param = sm.getPreloadParameter(); - string low_ktype = ktype; - to_lower(low_ktype); - HKU_CHECK(param.tryGet(low_ktype, false), "The ktype: {} is not be preloaded!", - ktype); - - json data; - const auto& jcodes = req["codes"]; - // HKU_INFO("codes size: {}", jcodes.size()); - for (auto iter = jcodes.cbegin(); iter != jcodes.cend(); ++iter) { - string market_code = to_string(*iter); - market_code = market_code.substr(1, market_code.size() - 2); - Stock stk = getStock(market_code); - if (stk.isNull()) { - HKU_WARN("Not found stock: {}", market_code); - continue; - } - - KRecordList krecords = - stk.getKRecordList(KQueryByIndex(-1, Null(), ktype)); - if (!krecords.empty()) { - const auto& k = krecords.back(); - json jr; - jr.emplace_back(market_code); - jr.emplace_back(k.datetime.str()); - jr.emplace_back(k.openPrice); - jr.emplace_back(k.highPrice); - jr.emplace_back(k.lowPrice); - jr.emplace_back(k.closePrice); - jr.emplace_back(k.transAmount); - jr.emplace_back(k.transCount); - data.emplace_back(std::move(jr)); - } - } - - json res; - res["data"] = data; - // HKU_INFO("<-- res: {}", to_string(res)); - return res; - }); - - server.start(); - // server.loop(); - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(10)); - } - - } catch (const std::exception& e) { - HKU_ERROR(e.what()); - } catch (...) { - HKU_ERROR("Unknown error!"); - } - - server.stop(); - -#if defined(_WIN32) - SetConsoleOutputCP(old_cp); -#endif - return 0; -} diff --git a/hikyuu_cpp/demo/xmake.lua b/hikyuu_cpp/demo/xmake.lua index 1953b42e..59fb328a 100644 --- a/hikyuu_cpp/demo/xmake.lua +++ b/hikyuu_cpp/demo/xmake.lua @@ -44,27 +44,3 @@ target("demo2") add_files("./demo2.cpp") target_end() - - -target("demo3") - set_kind("binary") - set_default(false) - - add_packages("boost", "spdlog", "fmt", "nng", "nlohmann_json") - add_includedirs("..") - - if is_plat("windows") then - add_cxflags("-wd4267") - add_cxflags("-wd4251") - end - - if is_plat("windows") and get_config("kind") == "shared" then - add_defines("HKU_API=__declspec(dllimport)") - add_defines("HKU_UTILS_API=__declspec(dllimport)") - add_defines("SQLITE_API=__declspec(dllimport)") - end - - add_deps("hikyuu") - - add_files("./demo3.cpp") -target_end()