mirror of
https://gitee.com/fasiondog/hikyuu.git
synced 2024-11-29 18:39:10 +08:00
398 lines
13 KiB
Python
398 lines
13 KiB
Python
# coding:utf-8
|
||
#
|
||
# The MIT License (MIT)
|
||
#
|
||
# Copyright (c) 2010-2019 fasiondog/hikyuu
|
||
#
|
||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
# of this software and associated documentation files (the "Software"), to deal
|
||
# in the Software without restriction, including without limitation the rights
|
||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||
# copies of the Software, and to permit persons to whom the Software is
|
||
# furnished to do so, subject to the following conditions:
|
||
#
|
||
# The above copyright notice and this permission notice shall be included in all
|
||
# copies or substantial portions of the Software.
|
||
#
|
||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||
# SOFTWARE.
|
||
|
||
import datetime
|
||
import tables as tb
|
||
|
||
from hikyuu.util import hku_catch
|
||
from hikyuu.util.mylog import hku_error, hku_debug
|
||
|
||
HDF5_COMPRESS_LEVEL = 9
|
||
|
||
|
||
class H5Record(tb.IsDescription):
|
||
"""HDF5基础K线数据格式（日线、分钟线、5分钟线"""
|
||
datetime = tb.UInt64Col() #IGNORE:E1101
|
||
openPrice = tb.UInt32Col() #IGNORE:E1101
|
||
highPrice = tb.UInt32Col() #IGNORE:E1101
|
||
lowPrice = tb.UInt32Col() #IGNORE:E1101
|
||
closePrice = tb.UInt32Col() #IGNORE:E1101
|
||
transAmount = tb.UInt64Col() #IGNORE:E1101
|
||
transCount = tb.UInt64Col() #IGNORE:E1101
|
||
|
||
|
||
class H5Index(tb.IsDescription):
|
||
"""HDF5扩展K线数据格式（周线、月线、季线、半年线、年线、15分钟线、30分钟线、60分钟线"""
|
||
datetime = tb.UInt64Col() #IGNORE:E1101
|
||
start = tb.UInt64Col() #IGNORE:E1101
|
||
|
||
|
||
class H5Transaction(tb.IsDescription):
|
||
"""分笔数据"""
|
||
datetime = tb.UInt64Col()
|
||
price = tb.UInt64Col()
|
||
vol = tb.UInt64Col()
|
||
buyorsell = tb.UInt8Col()
|
||
|
||
|
||
class H5TransactionIndex(tb.IsDescription):
|
||
"""分笔数据按天索引"""
|
||
datetime = tb.UInt64Col()
|
||
start = tb.UInt64Col()
|
||
|
||
|
||
class H5MinuteTime(tb.IsDescription):
|
||
"""分时线"""
|
||
datetime = tb.UInt64Col()
|
||
price = tb.UInt64Col()
|
||
vol = tb.UInt64Col()
|
||
|
||
|
||
#------------------------------------------------------------------------------
|
||
# K线数据
|
||
#------------------------------------------------------------------------------
|
||
|
||
|
||
def open_h5file(dest_dir, market, ktype):
|
||
filename = "{}/{}_{}.h5".format(dest_dir, market.lower(), ktype.lower())
|
||
h5file = tb.open_file(
|
||
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
|
||
)
|
||
return h5file
|
||
|
||
|
||
@hku_catch(None, trace=True)
|
||
def get_h5table(h5file, market, code):
|
||
try:
|
||
group = h5file.get_node("/", "data")
|
||
except:
|
||
group = h5file.create_group("/", "data")
|
||
|
||
tablename = market.upper() + code
|
||
try:
|
||
table = h5file.get_node(group, tablename)
|
||
except:
|
||
table = h5file.create_table(group, tablename, H5Record)
|
||
|
||
return table
|
||
|
||
|
||
def update_hdf5_extern_data(h5file, tablename, data_type):
|
||
"""更新周线、月线、15分钟线等扩展数据索引"""
|
||
def getWeekDate(olddate):
|
||
y = olddate // 100000000
|
||
m = olddate // 1000000 - y * 100
|
||
d = olddate // 10000 - (y * 10000 + m * 100)
|
||
tempdate = datetime.date(y, m, d)
|
||
# python中周一是第0天，周五的第4天
|
||
tempweekdate = tempdate + datetime.timedelta(4 - tempdate.weekday())
|
||
newdate = tempweekdate.year * 100000000 + tempweekdate.month * 1000000 + tempweekdate.day * 10000
|
||
return newdate
|
||
|
||
def getMonthDate(olddate):
|
||
y = olddate // 100000000
|
||
m = olddate // 1000000 - y * 100
|
||
import calendar
|
||
_, d = calendar.monthrange(y, m)
|
||
return (y * 100000000 + m * 1000000 + d * 10000)
|
||
|
||
def getQuarterDate(olddate):
|
||
quarterDict = {1: 3, 2: 3, 3: 3, 4: 6, 5: 6, 6: 6, 7: 9, 8: 9, 9: 9, 10: 12, 11: 12, 12: 12}
|
||
d_dict = {3: 310000, 6: 300000, 9: 300000, 12: 310000}
|
||
y = olddate // 100000000
|
||
m = olddate // 1000000 - y * 100
|
||
new_m = quarterDict[m]
|
||
return (y * 100000000 + new_m * 1000000 + d_dict[new_m])
|
||
|
||
def getHalfyearDate(olddate):
|
||
y = olddate // 100000000
|
||
m = olddate // 1000000 - y * 100
|
||
return y * 100000000 + (6300000 if m < 7 else 12310000)
|
||
|
||
def getYearDate(olddate):
|
||
y = olddate // 100000000
|
||
return (y * 100000000 + 12310000)
|
||
|
||
def getMin60Date(olddate):
|
||
mint = olddate - olddate // 10000 * 10000
|
||
if mint <= 1030:
|
||
newdate = olddate // 10000 * 10000 + 1030
|
||
elif mint <= 1130:
|
||
newdate = olddate // 10000 * 10000 + 1130
|
||
elif mint <= 1400:
|
||
newdate = olddate // 10000 * 10000 + 1400
|
||
else:
|
||
newdate = olddate // 10000 * 10000 + 1500
|
||
return newdate
|
||
def getHour2Date(olddate):
|
||
mint = olddate - olddate // 10000 * 10000
|
||
if mint <= 1130:
|
||
newdate = olddate // 10000 * 10000 + 1130
|
||
else:
|
||
newdate = olddate // 10000 * 10000 + 1500
|
||
|
||
return newdate
|
||
def getMin15Date(olddate):
|
||
mint = olddate - olddate // 10000 * 10000
|
||
if mint <= 945:
|
||
newdate = olddate // 10000 * 10000 + 945
|
||
elif mint <= 1000:
|
||
newdate = olddate // 10000 * 10000 + 1000
|
||
elif mint <= 1015:
|
||
newdate = olddate // 10000 * 10000 + 1015
|
||
elif mint <= 1030:
|
||
newdate = olddate // 10000 * 10000 + 1030
|
||
elif mint <= 1045:
|
||
newdate = olddate // 10000 * 10000 + 1045
|
||
elif mint <= 1100:
|
||
newdate = olddate // 10000 * 10000 + 1100
|
||
elif mint <= 1115:
|
||
newdate = olddate // 10000 * 10000 + 1115
|
||
elif mint <= 1130:
|
||
newdate = olddate // 10000 * 10000 + 1130
|
||
elif mint <= 1315:
|
||
newdate = olddate // 10000 * 10000 + 1315
|
||
elif mint <= 1330:
|
||
newdate = olddate // 10000 * 10000 + 1330
|
||
elif mint <= 1345:
|
||
newdate = olddate // 10000 * 10000 + 1345
|
||
elif mint <= 1400:
|
||
newdate = olddate // 10000 * 10000 + 1400
|
||
elif mint <= 1415:
|
||
newdate = olddate // 10000 * 10000 + 1415
|
||
elif mint <= 1430:
|
||
newdate = olddate // 10000 * 10000 + 1430
|
||
elif mint <= 1445:
|
||
newdate = olddate // 10000 * 10000 + 1445
|
||
else:
|
||
newdate = olddate // 10000 * 10000 + 1500
|
||
return newdate
|
||
|
||
def getMin30Date(olddate):
|
||
mint = olddate - olddate // 10000 * 10000
|
||
if mint <= 1000:
|
||
newdate = olddate // 10000 * 10000 + 1000
|
||
elif mint <= 1030:
|
||
newdate = olddate // 10000 * 10000 + 1030
|
||
elif mint <= 1100:
|
||
newdate = olddate // 10000 * 10000 + 1100
|
||
elif mint <= 1130:
|
||
newdate = olddate // 10000 * 10000 + 1130
|
||
elif mint <= 1330:
|
||
newdate = olddate // 10000 * 10000 + 1330
|
||
elif mint <= 1400:
|
||
newdate = olddate // 10000 * 10000 + 1400
|
||
elif mint <= 1430:
|
||
newdate = olddate // 10000 * 10000 + 1430
|
||
else:
|
||
newdate = olddate // 10000 * 10000 + 1500
|
||
return newdate
|
||
|
||
def getNewDate(index_type, olddate):
|
||
if index_type == 'week':
|
||
return getWeekDate(olddate)
|
||
elif index_type == 'month':
|
||
return getMonthDate(olddate)
|
||
elif index_type == 'quarter':
|
||
return getQuarterDate(olddate)
|
||
elif index_type == 'halfyear':
|
||
return getHalfyearDate(olddate)
|
||
elif index_type == 'year':
|
||
return getYearDate(olddate)
|
||
elif index_type == 'min15':
|
||
return getMin15Date(olddate)
|
||
elif index_type == 'min30':
|
||
return getMin30Date(olddate)
|
||
elif index_type == 'min60':
|
||
return getMin60Date(olddate)
|
||
elif index_type == 'hour2':
|
||
return getHour2Date(olddate)
|
||
else:
|
||
return None
|
||
|
||
if data_type == 'DAY':
|
||
index_list = ('week', 'month', 'quarter', 'halfyear', 'year')
|
||
else:
|
||
index_list = ('min15', 'min30', 'min60', 'hour2')
|
||
|
||
groupDict = {}
|
||
for index_type in index_list:
|
||
try:
|
||
groupDict[index_type] = h5file.get_node("/", index_type)
|
||
except:
|
||
groupDict[index_type] = h5file.create_group("/", index_type)
|
||
|
||
try:
|
||
table = h5file.get_node("/data", tablename)
|
||
except Exception as e:
|
||
hku_error("{}".format(e))
|
||
return
|
||
|
||
for index_type in index_list:
|
||
hku_debug("{} update {} index".format(tablename, index_type))
|
||
try:
|
||
index_table = h5file.get_node(groupDict[index_type], tablename)
|
||
except:
|
||
index_table = h5file.create_table(groupDict[index_type], tablename, H5Index)
|
||
|
||
total = table.nrows
|
||
if 0 == total:
|
||
continue
|
||
|
||
index_total = index_table.nrows
|
||
index_row = index_table.row
|
||
if index_total:
|
||
index_last_date = int(index_table[-1]['datetime'])
|
||
last_date = getNewDate(index_type, int(table[-1]['datetime']))
|
||
if index_last_date == last_date:
|
||
continue
|
||
startix = int(index_table[-1]['start'])
|
||
pre_index_date = int(index_table[-1]['datetime'])
|
||
else:
|
||
startix = 0
|
||
date = int(table[0]['datetime'])
|
||
pre_index_date = getNewDate(index_type, date)
|
||
index_row['datetime'] = pre_index_date
|
||
index_row['start'] = 0
|
||
index_row.append()
|
||
|
||
index = startix
|
||
for row in table[startix:]:
|
||
date = int(row['datetime'])
|
||
cur_index_date = getNewDate(index_type, date)
|
||
if cur_index_date != pre_index_date:
|
||
index_row['datetime'] = cur_index_date
|
||
index_row['start'] = index
|
||
index_row.append()
|
||
pre_index_date = cur_index_date
|
||
index += 1
|
||
index_table.flush()
|
||
|
||
|
||
#------------------------------------------------------------------------------
|
||
# 分笔数据
|
||
#------------------------------------------------------------------------------
|
||
|
||
|
||
def open_trans_file(dest_dir, market):
|
||
filename = "{}/{}_trans.h5".format(dest_dir, market.lower())
|
||
h5file = tb.open_file(
|
||
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
|
||
)
|
||
return h5file
|
||
|
||
|
||
@hku_catch(trace=True)
|
||
def get_trans_table(h5file, market, code):
|
||
try:
|
||
group = h5file.get_node("/", "data")
|
||
except:
|
||
group = h5file.create_group("/", "data")
|
||
|
||
tablename = market.upper() + code
|
||
try:
|
||
table = h5file.get_node(group, tablename)
|
||
except:
|
||
table = h5file.create_table(group, tablename, H5Transaction)
|
||
|
||
return table
|
||
|
||
|
||
def update_hdf5_trans_index(h5file, tablename):
|
||
"""更新分笔数据按日索引"""
|
||
hku_debug("{} update trans index".format(tablename))
|
||
try:
|
||
table = h5file.get_node("/data", tablename)
|
||
except:
|
||
return
|
||
|
||
total = table.nrows
|
||
if 0 == total:
|
||
return
|
||
|
||
try:
|
||
group = h5file.get_node("/", 'index')
|
||
except:
|
||
group = h5file.create_group("/", 'index')
|
||
|
||
try:
|
||
index_table = h5file.get_node(group, tablename)
|
||
except:
|
||
index_table = h5file.create_table(group, tablename, H5Index)
|
||
|
||
index_total = index_table.nrows
|
||
index_row = index_table.row
|
||
if index_total:
|
||
index_last_date = int(index_table[-1]['datetime'])
|
||
last_date = int(table[-1]['datetime'] // 1000000 * 10000)
|
||
if index_last_date == last_date:
|
||
return
|
||
startix = int(index_table[-1]['start'])
|
||
pre_index_date = index_last_date
|
||
else:
|
||
startix = 0
|
||
date = int(table[0]['datetime'] // 1000000 * 10000)
|
||
pre_index_date = date
|
||
index_row['datetime'] = pre_index_date
|
||
index_row['start'] = 0
|
||
index_row.append()
|
||
|
||
index = startix
|
||
for row in table[startix:]:
|
||
date = int(row['datetime'] // 1000000 * 10000)
|
||
cur_index_date = date
|
||
if cur_index_date != pre_index_date:
|
||
index_row['datetime'] = cur_index_date
|
||
index_row['start'] = index
|
||
index_row.append()
|
||
pre_index_date = cur_index_date
|
||
index += 1
|
||
index_table.flush()
|
||
|
||
|
||
#------------------------------------------------------------------------------
|
||
# 分时数据
|
||
#------------------------------------------------------------------------------
|
||
def open_time_file(dest_dir, market):
|
||
filename = "{}/{}_time.h5".format(dest_dir, market.lower())
|
||
h5file = tb.open_file(
|
||
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
|
||
)
|
||
return h5file
|
||
|
||
|
||
@hku_catch(None, trace=True)
|
||
def get_time_table(h5file, market, code):
|
||
try:
|
||
group = h5file.get_node("/", "data")
|
||
except:
|
||
group = h5file.create_group("/", "data")
|
||
|
||
tablename = market.upper() + code
|
||
try:
|
||
table = h5file.get_node(group, tablename)
|
||
except:
|
||
table = h5file.create_table(group, tablename, H5MinuteTime)
|
||
|
||
return table |