test/hikyuu2
1
0
Fork 0
mirror of https://gitee.com/fasiondog/hikyuu.git synced 2024-12-03 20:37:50 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

优化权息数据加载速度

Browse Source
This commit is contained in:
fasiondog 2024-01-05 21:24:45 +08:00
parent 0b915d42d4
commit 41866b78c8
8 changed files with 148 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hikyuu_cpp/hikyuu/StockManager.cpp
View File

@ -538,23 +538,15 @@ void StockManager::loadAllHolidays() {
void StockManager::loadAllStockWeights() {
HKU_INFO("Loading stock weight...");
ThreadPool tg; // 这里不用全局的线程池，可以避免在初始化后立即reload导致过长的等待
std::vector<std::future<void>> task_list;
auto all_stkweight_dict = m_baseInfoDriver->getAllStockWeightList();
std::lock_guard<std::mutex> lock(*m_stockDict_mutex);
for (auto iter = m_stockDict.begin(); iter != m_stockDict.end(); ++iter) {
task_list.push_back(tg.submit([=]() mutable {
auto weight_iter = all_stkweight_dict.find(iter->first);
if (weight_iter != all_stkweight_dict.end()) {
Stock& stock = iter->second;
StockWeightList weightList = m_baseInfoDriver->getStockWeightList(
stock.market(), stock.code(), Datetime::min(), Null<Datetime>());
if (stock.m_data) {
std::lock_guard<std::mutex> lock(stock.m_data->m_weight_mutex);
stock.m_data->m_weightList.swap(weightList);
}
}));
}
// 权息信息如果不等待加载完毕，在数据加载期间进行计算可能导致复权错误，所以这里需要等待
for (auto& task : task_list) {
task.get();
std::lock_guard<std::mutex> lock(stock.m_data->m_weight_mutex);
stock.m_data->m_weightList.swap(weight_iter->second);
}
}
}

5
hikyuu_cpp/hikyuu/data_driver/BaseInfoDriver.h
View File

@ -111,6 +111,11 @@ public:
virtual StockWeightList getStockWeightList(const string& market, const string& code,
Datetime start, Datetime end);
virtual unordered_map<string, StockWeightList> getAllStockWeightList() {
unordered_map<string, StockWeightList> ret;
return ret;
}
/**
* 获取当前财务信息
* @param market 市场标识

42
hikyuu_cpp/hikyuu/data_driver/base_info/mysql/MySQLBaseInfoDriver.cpp
View File

@ -138,6 +138,48 @@ StockWeightList MySQLBaseInfoDriver::getStockWeightList(const string &market, co
return result;
}
unordered_map<string, StockWeightList> MySQLBaseInfoDriver::getAllStockWeightList() {
unordered_map<string, StockWeightList> result;
HKU_ASSERT(m_pool);
try {
auto con = m_pool->getConnect();
HKU_CHECK(con, "Failed fetch connect!");
vector<StockWeightTableView> view;
con->batchLoadView(
view,
"SELECT a.id AS id, concat(market.market, stock.code) AS market_code, a.date, "
"a.countAsGift*0.0001 AS countAsGift, a.countForSell*0.0001 AS countForSell, "
"a.priceForSell*0.001 AS priceForSell, a.bonus*0.001,a.countOfIncreasement*0.0001 AS "
"countOfIncreasement, a.totalCount AS totalCount, a.freeCount AS freeCount FROM "
"stkweight AS a, stock, market WHERE a.stockid=stock.stockid AND "
"market.marketid=stock.marketid ORDER BY a.stockid, a.date");
for (const auto &record : view) {
auto iter = result.find(record.market_code);
if (iter == result.end()) {
auto in_iter = result.insert(std::make_pair(record.market_code, StockWeightList()));
if (in_iter.second) {
iter = in_iter.first;
}
}
iter->second.emplace_back(StockWeight(
Datetime(record.date), record.countAsGift, record.countForSell, record.priceForSell,
record.bonus, record.countOfIncreasement, record.totalCount, record.freeCount));
}
} catch (std::exception &e) {
HKU_FATAL("load StockWeight table failed! {}", e.what());
return result;
} catch (...) {
HKU_FATAL("load StockWeight table failed!");
return result;
}
return result;
}
vector<StockInfo> MySQLBaseInfoDriver::getAllStockInfo() {
vector<StockInfo> result;
HKU_ERROR_IF_RETURN(!m_pool, result, "Connect pool ptr is null!");

1
hikyuu_cpp/hikyuu/data_driver/base_info/mysql/MySQLBaseInfoDriver.h
View File

@ -34,6 +34,7 @@ public:
virtual Parameter getFinanceInfo(const string& market, const string& code) override;
virtual StockWeightList getStockWeightList(const string& market, const string& code,
Datetime start, Datetime end) override;
virtual unordered_map<string, StockWeightList> getAllStockWeightList() override;
virtual MarketInfo getMarketInfo(const string& market) override;
virtual StockTypeInfo getStockTypeInfo(uint32_t type) override;
virtual StockInfo getStockInfo(string market, const string& code) override;

42
hikyuu_cpp/hikyuu/data_driver/base_info/sqlite/SQLiteBaseInfoDriver.cpp
View File

@ -157,6 +157,48 @@ StockWeightList SQLiteBaseInfoDriver::getStockWeightList(const string& market, c
return result;
}
unordered_map<string, StockWeightList> SQLiteBaseInfoDriver::getAllStockWeightList() {
unordered_map<string, StockWeightList> result;
HKU_ASSERT(m_pool);
try {
auto con = m_pool->getConnect();
HKU_CHECK(con, "Failed fetch connect!");
vector<StockWeightTableView> view;
con->batchLoadView(
view,
"SELECT a.id AS id, (market.market || stock.code) AS market_code, a.date, "
"a.countAsGift*0.0001 AS countAsGift, a.countForSell*0.0001 AS countForSell, "
"a.priceForSell*0.001 AS priceForSell, a.bonus*0.001,a.countOfIncreasement*0.0001 AS "
"countOfIncreasement, a.totalCount AS totalCount, a.freeCount AS freeCount FROM "
"stkweight AS a, stock, market WHERE a.stockid=stock.stockid AND "
"market.marketid=stock.marketid ORDER BY a.stockid, a.date");
for (const auto& record : view) {
auto iter = result.find(record.market_code);
if (iter == result.end()) {
auto in_iter = result.insert(std::make_pair(record.market_code, StockWeightList()));
if (in_iter.second) {
iter = in_iter.first;
}
}
iter->second.emplace_back(StockWeight(
Datetime(record.date), record.countAsGift, record.countForSell, record.priceForSell,
record.bonus, record.countOfIncreasement, record.totalCount, record.freeCount));
}
} catch (std::exception& e) {
HKU_FATAL("load StockWeight table failed! {}", e.what());
return result;
} catch (...) {
HKU_FATAL("load StockWeight table failed!");
return result;
}
return result;
}
Parameter SQLiteBaseInfoDriver ::getFinanceInfo(const string& market, const string& code) {
Parameter result;
HKU_IF_RETURN(!m_pool, result);

3
hikyuu_cpp/hikyuu/data_driver/base_info/sqlite/SQLiteBaseInfoDriver.h
View File

@ -29,6 +29,7 @@ public:
virtual Parameter getFinanceInfo(const string& market, const string& code) override;
virtual StockWeightList getStockWeightList(const string& market, const string& code,
Datetime start, Datetime end) override;
virtual unordered_map<string, StockWeightList> getAllStockWeightList() override;
virtual MarketInfo getMarketInfo(const string& market) override;
virtual StockTypeInfo getStockTypeInfo(uint32_t type) override;
virtual StockInfo getStockInfo(string market, const string& code) override;
@ -36,7 +37,7 @@ public:
virtual std::unordered_set<Datetime> getAllHolidays() override;
private:
//股票基本信息数据库实例
// 股票基本信息数据库实例
ConnectPool<SQLiteConnect>* m_pool;
};

14
hikyuu_cpp/hikyuu/data_driver/base_info/table/StockWeightTable.h
View File

@ -31,6 +31,20 @@ public:
double freeCount{0.};
};
struct StockWeightTableView {
TABLE_BIND9(StockWeightTableView, stkweight, market_code, date, countAsGift, countForSell,
priceForSell, bonus, countOfIncreasement, totalCount, freeCount)
string market_code;
uint64_t date{0};
double countAsGift{0.};
double countForSell{0.};
double priceForSell{0.};
double bonus{0.};
double countOfIncreasement{0.};
double totalCount{0.};
double freeCount{0.};
};
} // namespace hku
#endif /* HIKYUU_DATA_DRIVER_BASE_INFO_TABLE_STOCKWEIGHTTABLE_H */

36
hikyuu_cpp/hikyuu/utilities/db_connect/DBConnectBase.h
View File

@ -139,6 +139,14 @@ class HKU_API DBConnectBase : public std::enable_shared_from_this<DBConnectBase>
template <typename T>
void load(T &item, const DBCondition &cond);
/**
* 加载模型数据至指定的模型实例, 仅供查询
* @param item 指定的模型实例
* @param sql 查询条件 select 的 sql 语句
*/
template <typename T>
void loadView(T &item, const std::string &sql);
/**
* 批量加载模型数据至容器（vector，list 等支持 push_back 的容器）
* @param container 指定容器
@ -155,6 +163,14 @@ class HKU_API DBConnectBase : public std::enable_shared_from_this<DBConnectBase>
template <typename Container>
void batchLoad(Container &container, const DBCondition &cond);
/**
* 批量加载模型数据至容器（vector，list 等支持 push_back 的容器）
* @param container 指定容器
* @param sql select 的查询语句
*/
template <typename Container>
void batchLoadView(Container &container, const std::string &sql);
/**
* 批量更新
* @param container 拥有迭代器的容器
@ -456,6 +472,26 @@ void DBConnectBase::batchLoad(Container &container, const DBCondition &cond) {
batchLoad(container, cond.str());
}
template <typename T>
void DBConnectBase::loadView(T &item, const std::string &sql) {
SQLStatementPtr st = getStatement(sql);
st->exec();
if (st->moveNext()) {
item.load(st);
}
}
template <typename Container>
void DBConnectBase::batchLoadView(Container &container, const std::string &sql) {
SQLStatementPtr st = getStatement(sql);
st->exec();
while (st->moveNext()) {
typename Container::value_type tmp;
tmp.load(st);
container.push_back(tmp);
}
}
template <class Container>
inline void DBConnectBase::batchUpdate(Container &container, bool autotrans) {
batchUpdate(container.begin(), container.end(), autotrans);