data tool(continue)

This commit is contained in:
fasiondog 2018-11-18 11:27:22 +08:00
parent 8b9fefb454
commit c95d1ad46d
9 changed files with 192 additions and 80 deletions

View File

@ -26,7 +26,7 @@ import os
import sqlite3
import urllib.request
from tdx_to_h5 import qianlong_import_weight
from weight_to_sqlite import qianlong_import_weight
class ImportWeightToSqliteTask:
def __init__(self, queue, sqlitefile, dest_dir):

View File

@ -25,7 +25,7 @@
import datetime
import tables as tb
HDF5_COMPRESS_LEVEL = 9
HDF5_COMPRESS_LEVEL = 6
class H5Record(tb.IsDescription):
"""HDF5基础K线数据格式日线、分钟线、5分钟线"""

View File

@ -63,7 +63,6 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.initUI()
self.initThreads()
def closeEvent(self, event):
if self.import_running:
QMessageBox.about(self, '', '')
@ -78,11 +77,9 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.escape_time_thread.stop()
event.accept()
def getUserConfigDir(self):
return os.path.expanduser('~') + '/.hikyuu'
def saveConfig(self):
current_config = self.getCurrentConfig()
filename = self.getUserConfigDir() + '/importdata-gui.ini'
@ -94,7 +91,6 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
with open(filename, 'w', encoding='utf-8') as f:
f.write(hikyuu_hdf5_config_template.format(dir=data_dir))
def initUI(self):
self.setWindowIcon(QIcon("./hikyuu.ico"))
self.setFixedSize(self.width(), self.height())
@ -103,7 +99,7 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.reset_progress_bar()
#读取保存的配置文件信息,如果不存在,则使用默认配置
this_dir = os.path.dirname(__file__)
this_dir = self.getUserConfigDir()
import_config = ConfigParser()
if os.path.exists(this_dir + '/importdata-gui.ini'):
import_config.read(this_dir + '/importdata-gui.ini', encoding = 'utf-8')
@ -164,7 +160,6 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.mysql_usr_lineEdit.setText(mysql_usr)
self.mysql_pwd_lineEdit.setText(mysql_pwd)
def getCurrentConfig(self):
import_config = ConfigParser()
import_config['quotation'] = {'stock': self.import_stock_checkBox.isChecked(),
@ -190,7 +185,6 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
'pwd': self.mysql_pwd_lineEdit.text()}
return import_config
def initThreads(self):
self.escape_time_thread = None
self.hdf5_import_thread = None
@ -208,7 +202,6 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.tdx_servers_comboBox.setEnabled(not tdx_enable)
self.tdx_port_lineEdit.setEnabled(not tdx_enable)
@pyqtSlot()
def on_select_tdx_dir_pushButton_clicked(self):
dlg = QFileDialog()

View File

@ -0,0 +1,30 @@
# coding:utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2010-2017 fasiondog/hikyuu
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import mysql.connector
from common import MARKETID, get_stktype_list
def create_database(connect):
"""创建数据库"""
pass

View File

@ -35,6 +35,7 @@ from sqlite3_common import (get_codepre_list, create_database,
from h5_common import (H5Record, H5Index,
open_h5file, get_h5table,
update_hdf5_extern_data)
from weight_to_sqlite import qianlong_import_weight
def ProgressBar(cur, total):
@ -115,6 +116,7 @@ def import_stock_name(connect, api, market, quotations=None):
#print('%s新增股票数%i' % (market.upper(), count))
connect.commit()
cur.close()
return count
def guess_day_n_step(last_datetime):
@ -275,6 +277,8 @@ def import_data(connect, market, ktype, quotations, api, dest_dir, progress=Prog
total = len(stock_list)
for i, stock in enumerate(stock_list):
if stock[3] == 0:
if progress:
progress(i, total)
continue
this_count = import_one_stock_data(connect, api, h5file, market, ktype, stock)
@ -293,13 +297,14 @@ def import_data(connect, market, ktype, quotations, api, dest_dir, progress=Prog
if __name__ == '__main__':
import os
import time
starttime = time.time()
dest_dir = "c:\\stock"
tdx_server = '119.147.212.81'
tdx_port = 7709
quotations = ['stock', 'fund', 'bond']
quotations = ['stock', 'fund']
connect = sqlite3.connect(dest_dir + "\\stock.db")
create_database(connect)
@ -308,20 +313,55 @@ if __name__ == '__main__':
api = TdxHq_API()
api.connect(tdx_server, tdx_port)
print("导入股票代码表")
#import_stock_name(connect, api, 'SH', quotations)
#import_stock_name(connect, api, 'SZ', quotations)
add_count = 0
"""
print("导入股票代码表")
add_count = import_stock_name(connect, api, 'SH', quotations)
add_count += import_stock_name(connect, api, 'SZ', quotations)
print("新增股票数:", add_count)
print("\n导入上证日线数据")
#add_count = import_data(connect, 'SH', 'DAY', ['bond'], api, dest_dir, progress=ProgressBar)
add_count = import_data(connect, 'SH', 'DAY', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入深证日线数据")
#add_count = import_data(connect, 'SZ', 'DAY', ['stock'], api, dest_dir, progress=ProgressBar)
add_count = import_data(connect, 'SZ', 'DAY', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入上证5分钟线数据")
add_count = import_data(connect, 'SH', '5MIN', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入深证5分钟线数据")
add_count = import_data(connect, 'SZ', '5MIN', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入上证分钟线数据")
add_count = import_data(connect, 'SH', '1MIN', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入深证分钟线数据")
add_count = import_data(connect, 'SZ', '1MIN', quotations, api, dest_dir, progress=ProgressBar)
print("\n导入数量:", add_count)
print("\n导入权息数据")
print("正在下载权息数据...")
import urllib.request
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60)
dest_filename = dest_dir + '/weight.rar'
with open(dest_filename, 'wb') as file:
file.write(net_file.read())
print("下载完成,正在解压...")
os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir))
print("解压完成,正在导入...")
add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH')
add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ')
print("导入数量:", add_count)
"""
for i in range(10):
x = api.get_history_transaction_data(TDXParams.MARKET_SZ, '000001', (9-i)*2000, 2000, 20181112)
#x = api.get_transaction_data(TDXParams.MARKET_SZ, '000001', (9-i)*800, 800)
@ -330,11 +370,9 @@ if __name__ == '__main__':
api.disconnect()
connect.close()
endtime = time.time()
print("\nTotal time:")
print("%.2fs" % (endtime - starttime))
print("%.2fm" % ((endtime - starttime) / 60))

View File

@ -32,13 +32,14 @@ def create_database(connect):
"""创建SQLITE3数据库表"""
try:
cur = connect.cursor()
filename = os.path.dirname(__file__) + '/sqlite_createdb.sql'
filename = os.path.dirname(__file__) + '/sqlite_upgrade/createdb.sql'
with open(filename, 'r', encoding='utf8') as sqlfile:
cur.executescript(sqlfile.read())
connect.commit()
cur.close()
except sqlite3.OperationalError:
print("相关数据表可能已经存在,放弃创建。如需重建,请手工删除相关数据表。")
print("相关数据表可能已经存在,忽略创建。如需重建,请手工删除相关数据表。")
pass
except Exception as e:
raise(e)

View File

@ -23,7 +23,6 @@
# SOFTWARE.
import os.path
import pathlib
import struct
import sqlite3
import datetime
@ -40,6 +39,7 @@ from sqlite3_common import (create_database, get_marketid,
from h5_common import (H5Record, H5Index,
open_h5file, get_h5table,
update_hdf5_extern_data)
from weight_to_sqlite import qianlong_import_weight
def ProgressBar(cur, total):
@ -121,6 +121,7 @@ def tdx_import_stock_name_from_file(connect, filename, market, quotations=None):
#print('%s新增股票数%i' % (market.upper(), count))
connect.commit()
cur.close()
return count
def tdx_import_day_data_from_file(connect, filename, h5file, market, stock_record):
@ -331,6 +332,8 @@ def tdx_import_data(connect, market, ktype, quotations, src_dir, dest_dir, progr
total = len(a)
for i, stock in enumerate(a):
if stock[3] == 0:
if progress:
progress(i, total)
continue
filename = src_dir + "\\" + market.lower() + stock[2]+ suffix
@ -349,52 +352,6 @@ def tdx_import_data(connect, market, ktype, quotations, src_dir, dest_dir, progr
return add_record_count
def qianlong_import_weight(connect, src_dir, market):
"""导入钱龙格式的权息数据"""
cur = connect.cursor()
marketid = cur.execute("select marketid from Market where market='%s'" % market)
marketid = [id[0] for id in marketid]
marketid = marketid[0]
src_path = pathlib.Path(src_dir + '/shase/weight') if market == 'SH' else pathlib.Path(src_dir + '/sznse/weight')
wgt_file_list = list(src_path.glob('*.wgt'))
total_count = 0
for wgt_file in wgt_file_list:
code = wgt_file.stem
stockid = cur.execute("select stockid from Stock where marketid=%s and code='%s'" % (marketid, code))
stockid = [id[0] for id in stockid]
if not stockid:
continue
with wgt_file.open('rb') as sourcefile:
stockid = stockid[0]
a = cur.execute("select date from stkweight where stockid=%s" % stockid)
dateDict = dict([(i[0], None) for i in a])
records = []
data = sourcefile.read(36)
while data:
a = struct.unpack('iiiiiiiii', data)
date = (a[0] >> 20) * 10000 + (((a[0] << 12) & 4294967295) >> 28) * 100 + ((a[0] & 0xffff) >> 11)
if date not in dateDict:
records.append((stockid, date, a[1], a[2], a[3], a[4], a[5], a[6], a[7]))
data = sourcefile.read(36)
sourcefile.close()
if records:
cur.executemany("INSERT INTO StkWeight(stockid, date, countAsGift, \
countForSell, priceForSell, bonus, countOfIncreasement, totalCount, freeCount) \
VALUES (?,?,?,?,?,?,?,?,?)",
records)
total_count += len(records)
connect.commit()
cur.close()
return total_count
if __name__ == '__main__':
import time
@ -402,34 +359,57 @@ if __name__ == '__main__':
src_dir = "D:\\TdxW_HuaTai"
dest_dir = "c:\\stock"
quotations = ['stock', 'fund', 'bond'] #
quotations = ['stock', 'fund'] #
connect = sqlite3.connect(dest_dir + "\\stock.db")
create_database(connect)
add_count = 0
print("导入股票代码表")
tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\shm.tnf", 'SH', quotations)
tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\szm.tnf", 'SZ', quotations)
add_count = tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\shm.tnf", 'SH', quotations)
add_count += tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\szm.tnf", 'SZ', quotations)
print("新增股票数:", add_count)
print("\n导入上证日线数据")
add_count = 0
#add_count = tdx_import_data(connect, 'SH', 'DAY', ['stock', 'fund'], src_dir + "\\vipdoc\\sh\\lday", dest_dir)
#add_count = tdx_import_data(connect, 'SZ', 'DAY', 'stock', src_dir + "\\vipdoc\\sz\\lday", dest_dir)
add_count = tdx_import_data(connect, 'SH', 'DAY', quotations, src_dir + "\\vipdoc\\sh\\lday", dest_dir)
print("\n导入数量:", add_count)
print("\n导入深证日线数据")
add_count = tdx_import_data(connect, 'SZ', 'DAY', quotations, src_dir + "\\vipdoc\\sz\\lday", dest_dir)
print("\n导入数量:", add_count)
print("\n导入上证5分钟数据")
#add_count = tdx_import_data(connect, 'SH', '5MIN', 'stock', src_dir + "\\vipdoc\\sh\\fzline", dest_dir)
add_count = tdx_import_data(connect, 'SH', '5MIN', quotations, src_dir + "\\vipdoc\\sh\\fzline", dest_dir)
print("\n导入数量:", add_count)
print("\n导入深证5分钟数据")
add_count = tdx_import_data(connect, 'SZ', '5MIN', quotations, src_dir + "\\vipdoc\\sz\\fzline", dest_dir)
print("\n导入数量:", add_count)
print("\n导入上证1分钟数据")
#add_count = tdx_import_data(connect, 'SH', '1MIN', 'stock', src_dir + "\\vipdoc\\sh\\minline", dest_dir)
add_count = tdx_import_data(connect, 'SH', '1MIN', quotations, src_dir + "\\vipdoc\\sh\\minline", dest_dir)
print("\n导入数量:", add_count)
print("\n导入深证1分钟数据")
add_count = tdx_import_data(connect, 'SZ', '1MIN', quotations, src_dir + "\\vipdoc\\sz\\minline", dest_dir)
print("\n导入数量:", add_count)
print("\n导入权息数据")
total_count = 0
#total_count = qianlong_import_weight(connect, r'C:\stock\weight', 'SH')
#total_count += qianlong_import_weight(connect, r'C:\stock\weight', 'SZ')
print(total_count)
print("正在下载权息数据...")
import urllib.request
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60)
dest_filename = dest_dir + '/weight.rar'
with open(dest_filename, 'wb') as file:
file.write(net_file.read())
print("下载完成,正在解压...")
os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir))
print("解压完成,正在导入...")
add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH')
add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ')
print("导入数量:", add_count)
connect.close()

View File

@ -0,0 +1,70 @@
# coding:utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2010-2017 fasiondog/hikyuu
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import struct
import pathlib
def qianlong_import_weight(connect, src_dir, market):
"""导入钱龙格式的权息数据"""
cur = connect.cursor()
marketid = cur.execute("select marketid from Market where market='%s'" % market)
marketid = [id[0] for id in marketid]
marketid = marketid[0]
src_path = pathlib.Path(src_dir + '/shase/weight') if market == 'SH' else pathlib.Path(src_dir + '/sznse/weight')
wgt_file_list = list(src_path.glob('*.wgt'))
total_count = 0
for wgt_file in wgt_file_list:
code = wgt_file.stem
stockid = cur.execute("select stockid from Stock where marketid=%s and code='%s'" % (marketid, code))
stockid = [id[0] for id in stockid]
if not stockid:
continue
with wgt_file.open('rb') as sourcefile:
stockid = stockid[0]
a = cur.execute("select date from stkweight where stockid=%s" % stockid)
dateDict = dict([(i[0], None) for i in a])
records = []
data = sourcefile.read(36)
while data:
a = struct.unpack('iiiiiiiii', data)
date = (a[0] >> 20) * 10000 + (((a[0] << 12) & 4294967295) >> 28) * 100 + ((a[0] & 0xffff) >> 11)
if date not in dateDict:
records.append((stockid, date, a[1], a[2], a[3], a[4], a[5], a[6], a[7]))
data = sourcefile.read(36)
sourcefile.close()
if records:
cur.executemany("INSERT INTO StkWeight(stockid, date, countAsGift, \
countForSell, priceForSell, bonus, countOfIncreasement, totalCount, freeCount) \
VALUES (?,?,?,?,?,?,?,?,?)",
records)
total_count += len(records)
connect.commit()
cur.close()
return total_count