mirror of
https://gitee.com/fasiondog/hikyuu.git
synced 2024-12-02 03:48:19 +08:00
更新数据导入工具(continue)
This commit is contained in:
fasiondog
parent
92a4b176d6
commit
978f8fa803
201
hikyuu/data/pytdx_weight_to_mysql.py
Normal file
201
hikyuu/data/pytdx_weight_to_mysql.py
Normal file
@ -0,0 +1,201 @@
|
||||
# coding:utf-8
|
||||
#
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2010-2019 fasiondog/hikyuu
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from pytdx.hq import TDXParams
|
||||
import mysql.connector
|
||||
|
||||
|
||||
def to_pytdx_market(market):
|
||||
"""转换为pytdx的market"""
|
||||
pytdx_market = {'SH': TDXParams.MARKET_SH, 'SZ': TDXParams.MARKET_SZ}
|
||||
return pytdx_market[market.upper()]
|
||||
|
||||
|
||||
def pytdx_import_weight_to_mysql(pytdx_api, connect, market):
|
||||
"""导入钱龙格式的权息数据"""
|
||||
cur = connect.cursor()
|
||||
cur.execute("select marketid from `hku_base`.`Market` where market='%s'" % market)
|
||||
marketid = [id[0] for id in cur.fetchall()]
|
||||
marketid = marketid[0]
|
||||
pytdx_market = to_pytdx_market(market)
|
||||
|
||||
total_count = 0
|
||||
cur.execute("select stockid, code from `hku_base`.`Stock` where marketid=%s" % (marketid))
|
||||
stockid_list = [x for x in cur.fetchall()]
|
||||
cur.close()
|
||||
|
||||
for stockrecord in stockid_list:
|
||||
stockid, code = stockrecord
|
||||
#print("{}{}".format(market, code))
|
||||
|
||||
# 获取当前数据库中最后的一条权息记录的总股本和流通股本
|
||||
cur = connect.cursor()
|
||||
cur.execute(
|
||||
"select id, stockid, date, countAsGift, countForSell, priceForSell, \
|
||||
bonus, countOfIncreasement, totalCount, \
|
||||
freeCount from `hku_base`.`stkweight` where stockid=%s \
|
||||
order by date desc limit 1" % stockid
|
||||
)
|
||||
a = [x for x in cur.fetchall()]
|
||||
last_db_weight = None
|
||||
if a:
|
||||
a = list(a[0])
|
||||
last_db_weight = a
|
||||
db_last_date = a[2]
|
||||
last_total_count, last_free_count = a[8:10]
|
||||
else:
|
||||
last_db_weight = None
|
||||
db_last_date, last_total_count, last_free_count = (0, 0, 0)
|
||||
cur.close()
|
||||
|
||||
xdxr_list = pytdx_api.get_xdxr_info(pytdx_market, code)
|
||||
update_last_db_weight = False
|
||||
new_last_db_weight = last_db_weight
|
||||
records = {}
|
||||
for xdxr in xdxr_list:
|
||||
try:
|
||||
date = xdxr['year'] * 1000 + xdxr['month'] * 100 + xdxr['day']
|
||||
if date < db_last_date:
|
||||
continue
|
||||
if date == db_last_date and new_last_db_weight is not None:
|
||||
if xdxr['songzhuangu'] is not None:
|
||||
new_last_db_weight[3] = int(10000 * xdxr['songzhuangu'])
|
||||
update_last_db_weight = True
|
||||
if xdxr['peigu'] is not None:
|
||||
new_last_db_weight[4] = int(10000 * xdxr['peigu'])
|
||||
update_last_db_weight = True
|
||||
if xdxr['peigujia'] is not None:
|
||||
new_last_db_weight[5] = int(1000 * xdxr['peigujia'])
|
||||
update_last_db_weight = True
|
||||
if xdxr['fenhong'] is not None:
|
||||
new_last_db_weight[6] = int(1000 * xdxr['fenhong'])
|
||||
update_last_db_weight = True
|
||||
if xdxr['houzongguben'] is not None:
|
||||
new_last_db_weight[8] = round(xdxr['houzongguben'])
|
||||
update_last_db_weight = True
|
||||
last_total_count = new_last_db_weight[8]
|
||||
if xdxr['panhouliutong'] is not None:
|
||||
new_last_db_weight[9] = round(xdxr['panhouliutong'])
|
||||
update_last_db_weight = True
|
||||
last_free_count = new_last_db_weight[9]
|
||||
continue
|
||||
if date not in records:
|
||||
records[date] = [
|
||||
stockid,
|
||||
date,
|
||||
int(10000 * xdxr['songzhuangu'])
|
||||
if xdxr['songzhuangu'] is not None else 0, #countAsGift
|
||||
int(10000 *
|
||||
xdxr['peigu']) if xdxr['peigu'] is not None else 0, #countForSell
|
||||
int(1000 *
|
||||
xdxr['peigujia']) if xdxr['peigujia'] is not None else 0, #priceForSell
|
||||
int(1000 * xdxr['fenhong']) if xdxr['fenhong'] is not None else 0, #bonus
|
||||
0, #countOfIncreasement, pytdx 不区分送股和转增股，统一记在送股
|
||||
round(xdxr['houzongguben'])
|
||||
if xdxr['houzongguben'] is not None else last_total_count, #totalCount
|
||||
round(xdxr['panhouliutong'])
|
||||
if xdxr['panhouliutong'] is not None else last_free_count #freeCount
|
||||
]
|
||||
else:
|
||||
if xdxr['songzhuangu'] is not None:
|
||||
records[date][2] = int(10000 * xdxr['songzhuangu'])
|
||||
if xdxr['peigu'] is not None:
|
||||
records[date][3] = int(10000 * xdxr['peigu'])
|
||||
if xdxr['peigujia'] is not None:
|
||||
records[date][4] = int(1000 * xdxr['peigujia'])
|
||||
if xdxr['fenhong'] is not None:
|
||||
records[date][5] = int(1000 * xdxr['fenhong'])
|
||||
if xdxr['houzongguben'] is not None:
|
||||
records[date][7] = round(xdxr['houzongguben'])
|
||||
if xdxr['panhouliutong'] is not None:
|
||||
records[date][8] = round(xdxr['panhouliutong'])
|
||||
if xdxr['houzongguben'] is not None:
|
||||
last_total_count = round(xdxr['houzongguben'])
|
||||
if xdxr['panhouliutong'] is not None:
|
||||
last_free_count = round(xdxr['panhouliutong'])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(
|
||||
"{} {}{} xdxr: {} last_db_weigth:{}".format(
|
||||
stockid, market, code, xdxr, new_last_db_weight
|
||||
)
|
||||
)
|
||||
raise e
|
||||
|
||||
if update_last_db_weight:
|
||||
cur = connect.cursor()
|
||||
x = new_last_db_weight
|
||||
cur.execute(
|
||||
"UPDATE `hku_base`.`StkWeight` SET countAsGift=%s, countForSell=%s, priceForSell=%s, \
|
||||
bonus=%s, totalCount=%s, freeCount=%s \
|
||||
where id=%s" % (x[3], x[4], x[5], x[6], x[8], x[9], x[0])
|
||||
)
|
||||
connect.commit()
|
||||
cur.close()
|
||||
|
||||
if records:
|
||||
cur = connect.cursor()
|
||||
cur.executemany(
|
||||
"INSERT INTO `hku_base`.`StkWeight` (stockid, date, countAsGift, \
|
||||
countForSell, priceForSell, bonus, countOfIncreasement, totalCount, freeCount) \
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", [x for x in records.values()]
|
||||
)
|
||||
connect.commit()
|
||||
cur.close()
|
||||
total_count += len(records)
|
||||
|
||||
return total_count
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
import time
|
||||
import sqlite3
|
||||
from hikyuu.data.common_mysql import create_database
|
||||
starttime = time.time()
|
||||
|
||||
host = '127.0.0.1'
|
||||
port = 3306
|
||||
usr = 'root'
|
||||
pwd = 'Hikyuu2020*'
|
||||
tdx_server = '119.147.212.81'
|
||||
tdx_port = 7709
|
||||
quotations = ['stock', 'fund']
|
||||
|
||||
connect = mysql.connector.connect(user=usr, password=pwd, host=host, port=port)
|
||||
create_database(connect)
|
||||
|
||||
from pytdx.hq import TdxHq_API, TDXParams
|
||||
api = TdxHq_API()
|
||||
api.connect(tdx_server, tdx_port)
|
||||
|
||||
print("导入数量: {}".format(pytdx_import_weight_to_mysql(api, connect, 'SH')))
|
||||
|
||||
api.disconnect()
|
||||
connect.close()
|
||||
|
||||
endtime = time.time()
|
||||
print("\nTotal time:")
|
||||
print("%.2fs" % (endtime - starttime))
|
||||
print("%.2fm" % ((endtime - starttime) / 60))
|
||||
@ -11,6 +11,10 @@ from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QMessageBox
|
||||
from PyQt5.QtCore import pyqtSlot
|
||||
from PyQt5.QtGui import QIcon
|
||||
|
||||
import mysql.connector
|
||||
from mysql.connector import errorcode
|
||||
from mysql.connector.locales.eng import client_error #此句仅为pyinstaller打包时能够自动引入
|
||||
|
||||
from hikyuu.gui.data.MainWindow import *
|
||||
from hikyuu.gui.data.EscapetimeThread import EscapetimeThread
|
||||
from hikyuu.gui.data.UseTdxImportToH5Thread import UseTdxImportToH5Thread
|
||||
@ -295,6 +299,30 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
|
||||
self.mysql_pwd_lineEdit.setEnabled(mysql_enable)
|
||||
self.mysql_test_pushButton.setEnabled(mysql_enable)
|
||||
|
||||
@pyqtSlot()
|
||||
def on_mysql_test_pushButton_clicked(self):
|
||||
"""测试数据库连接"""
|
||||
db_config = {
|
||||
'user': self.mysql_usr_lineEdit.text(),
|
||||
'password': self.mysql_pwd_lineEdit.text(),
|
||||
'host': self.mysql_ip_lineEdit.text(),
|
||||
'port': self.mysql_port_lineEdit.text()
|
||||
}
|
||||
|
||||
try:
|
||||
cnx = mysql.connector.connect(**db_config)
|
||||
cnx.close()
|
||||
except mysql.connector.Error as err:
|
||||
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
|
||||
QMessageBox.critical(self, "测试数据库连接", "MYSQL密码或用户名错误！")
|
||||
elif err.errno == errorcode.ER_BAD_DB_ERROR:
|
||||
QMessageBox.critical(self, "测试数据库连接", "MySQL数据库不存在！")
|
||||
else:
|
||||
QMessageBox.critical(self, "测试数据库连接", err.msg)
|
||||
return
|
||||
|
||||
QMessageBox.about(self, "测试数据库连接", " 连接成功！")
|
||||
|
||||
def reset_progress_bar(self):
|
||||
self.hdf5_weight_label.setText('')
|
||||
self.hdf5_day_progressBar.setValue(0)
|
||||
|
||||
@ -23,8 +23,10 @@
|
||||
# SOFTWARE.
|
||||
|
||||
import sqlite3
|
||||
import mysql.connector
|
||||
from pytdx.hq import TdxHq_API
|
||||
from hikyuu.data.pytdx_to_h5 import import_data
|
||||
from hikyuu.data.pytdx_to_h5 import import_data as h5_import_data
|
||||
from hikyuu.data.pytdx_to_mysql import import_data as mysql_import_data
|
||||
|
||||
|
||||
class ProgressBar:
|
||||
@ -39,11 +41,11 @@ class ProgressBar:
|
||||
|
||||
class ImportPytdxToH5:
|
||||
def __init__(
|
||||
self, queue, sqlitefile, market, ktype, quotations, ip, port, dest_dir, start_datetime
|
||||
self, queue, config, market, ktype, quotations, ip, port, dest_dir, start_datetime
|
||||
):
|
||||
self.task_name = 'IMPORT_KDATA'
|
||||
self.queue = queue
|
||||
self.sqlitefile = sqlitefile
|
||||
self.config = config
|
||||
self.market = market
|
||||
self.ktype = ktype
|
||||
self.quotations = quotations
|
||||
@ -51,15 +53,27 @@ class ImportPytdxToH5:
|
||||
self.port = port
|
||||
self.dest_dir = dest_dir
|
||||
self.startDatetime = start_datetime
|
||||
self.import_data = h5_import_data
|
||||
|
||||
def __call__(self):
|
||||
if self.config['hdf5']['enable']:
|
||||
sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
|
||||
connect = sqlite3.connect(sqlite_file, timeout=1800)
|
||||
else:
|
||||
db_config = {
|
||||
'user': self.config['mysql']['usr'],
|
||||
'password': self.config['mysql']['pwd'],
|
||||
'host': self.config['mysql']['ip'],
|
||||
'port': self.config['mysql']['port']
|
||||
}
|
||||
connect = mysql.connector.connect(**db_config)
|
||||
|
||||
count = 0
|
||||
connect = sqlite3.connect(self.sqlitefile, timeout=1800)
|
||||
try:
|
||||
progress = ProgressBar(self)
|
||||
api = TdxHq_API()
|
||||
api.connect(self.ip, self.port)
|
||||
count = import_data(
|
||||
count = self.import_data(
|
||||
connect, self.market, self.ktype, self.quotations, api, self.dest_dir,
|
||||
self.startDatetime, progress
|
||||
)
|
||||
|
||||
@ -32,10 +32,13 @@ from hikyuu.gui.data.ImportPytdxTransToH5Task import ImportPytdxTransToH5
|
||||
from hikyuu.gui.data.ImportPytdxTimeToH5Task import ImportPytdxTimeToH5
|
||||
from hikyuu.gui.data.ImportHistoryFinanceTask import ImportHistoryFinanceTask
|
||||
from pytdx.hq import TdxHq_API
|
||||
from hikyuu.data.common_sqlite3 import create_database
|
||||
from hikyuu.data.pytdx_to_h5 import import_stock_name
|
||||
from hikyuu.data.common_pytdx import search_best_tdx
|
||||
|
||||
from hikyuu.data.common_sqlite3 import create_database as sqlite_create_database
|
||||
from hikyuu.data.pytdx_to_h5 import import_stock_name as sqlite_import_stock_name
|
||||
from hikyuu.data.common_mysql import create_database as mysql_create_database
|
||||
from hikyuu.data.pytdx_to_mysql import import_stock_name as mysql_import_stock_name
|
||||
|
||||
|
||||
class UsePytdxImportToH5Thread(QThread):
|
||||
message = pyqtSignal(list)
|
||||
@ -57,6 +60,13 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
|
||||
self.queue = Queue()
|
||||
|
||||
if config['hdf5']['enable']:
|
||||
self.create_database = sqlite_create_database
|
||||
self.import_stock_name = sqlite_import_stock_name
|
||||
else:
|
||||
self.create_database = mysql_create_database
|
||||
self.import_stock_name = mysql_import_stock_name
|
||||
|
||||
def __del__(self):
|
||||
for p in self.process_list:
|
||||
if p.is_alive():
|
||||
@ -74,8 +84,8 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
if self.config.getboolean('weight', 'enable', fallback=False):
|
||||
self.tasks.append(ImportWeightToSqliteTask(self.queue, sqlite_file_name, dest_dir))
|
||||
|
||||
if self.config.getboolean('finance', 'enable', fallback=False):
|
||||
self.tasks.append(ImportHistoryFinanceTask(self.queue, dest_dir))
|
||||
#if self.config.getboolean('finance', 'enable', fallback=False):
|
||||
# self.tasks.append(ImportHistoryFinanceTask(self.queue, dest_dir))
|
||||
|
||||
task_count = 0
|
||||
if self.config.getboolean('ktype', 'day', fallback=False):
|
||||
@ -141,19 +151,17 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
'%Y-%m-%d').date()
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SH', '1MIN', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SH', '1MIN', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SZ', '1MIN', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SZ', '1MIN', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
@ -184,19 +192,17 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
'%Y-%m-%d').date()
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SH', '5MIN', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SH', '5MIN', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SZ', '5MIN', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SZ', '5MIN', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
@ -206,19 +212,17 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
'%Y-%m-%d').date()
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SH', 'DAY', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SH', 'DAY', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
self.tasks.append(
|
||||
ImportPytdxToH5(
|
||||
self.queue, sqlite_file_name, 'SZ', 'DAY', self.quotations,
|
||||
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
|
||||
start_date.year * 100000000 + start_date.month * 1000000 +
|
||||
start_date.day * 10000
|
||||
self.queue, self.config, 'SZ', 'DAY', self.quotations, use_hosts[cur_host][0],
|
||||
use_hosts[cur_host][1], dest_dir, start_date.year * 100000000 +
|
||||
start_date.month * 1000000 + start_date.day * 10000
|
||||
)
|
||||
)
|
||||
cur_host += 1
|
||||
@ -254,15 +258,15 @@ class UsePytdxImportToH5Thread(QThread):
|
||||
self.send_message(['INFO', '导入股票代码表'])
|
||||
|
||||
connect = sqlite3.connect(dest_dir + "/stock.db")
|
||||
create_database(connect)
|
||||
self.create_database(connect)
|
||||
|
||||
pytdx_api = TdxHq_API()
|
||||
pytdx_api.connect(self.hosts[0][2], self.hosts[0][3])
|
||||
|
||||
count = import_stock_name(connect, pytdx_api, 'SH', self.quotations)
|
||||
count = self.import_stock_name(connect, pytdx_api, 'SH', self.quotations)
|
||||
if count > 0:
|
||||
self.send_message(['INFO', '上证新增股票数：%s' % count])
|
||||
count = import_stock_name(connect, pytdx_api, 'SZ', self.quotations)
|
||||
count = self.import_stock_name(connect, pytdx_api, 'SZ', self.quotations)
|
||||
if count > 0:
|
||||
self.send_message(['INFO', '深证新增股票数：%s' % count])
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user