hikyuu2/hikyuu/data/common_h5.py

398 lines
13 KiB
C++
Raw Normal View History

2018-11-12 02:33:56 +08:00
# coding:utf-8
#
# The MIT License (MIT)
#
2019-02-24 19:36:47 +08:00
# Copyright (c) 2010-2019 fasiondog/hikyuu
2018-11-12 02:33:56 +08:00
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import datetime
import tables as tb
2022-01-04 00:25:55 +08:00
from hikyuu.util import hku_catch
from hikyuu.util.mylog import hku_error, hku_debug
2022-01-04 00:25:55 +08:00
2018-11-28 01:49:17 +08:00
HDF5_COMPRESS_LEVEL = 9
2018-11-12 02:33:56 +08:00
2022-01-04 00:25:55 +08:00
2018-11-12 02:33:56 +08:00
class H5Record(tb.IsDescription):
"""HDF5基础K线数据格式日线、分钟线、5分钟线"""
2022-01-04 00:25:55 +08:00
datetime = tb.UInt64Col() #IGNORE:E1101
openPrice = tb.UInt32Col() #IGNORE:E1101
highPrice = tb.UInt32Col() #IGNORE:E1101
lowPrice = tb.UInt32Col() #IGNORE:E1101
closePrice = tb.UInt32Col() #IGNORE:E1101
transAmount = tb.UInt64Col() #IGNORE:E1101
transCount = tb.UInt64Col() #IGNORE:E1101
2018-11-12 02:33:56 +08:00
class H5Index(tb.IsDescription):
"""HDF5扩展K线数据格式周线、月线、季线、半年线、年线、15分钟线、30分钟线、60分钟线"""
2022-01-04 00:25:55 +08:00
datetime = tb.UInt64Col() #IGNORE:E1101
start = tb.UInt64Col() #IGNORE:E1101
2018-11-12 02:33:56 +08:00
2018-11-25 16:46:07 +08:00
class H5Transaction(tb.IsDescription):
"""分笔数据"""
datetime = tb.UInt64Col()
price = tb.UInt64Col()
vol = tb.UInt64Col()
buyorsell = tb.UInt8Col()
class H5TransactionIndex(tb.IsDescription):
"""分笔数据按天索引"""
datetime = tb.UInt64Col()
start = tb.UInt64Col()
class H5MinuteTime(tb.IsDescription):
"""分时线"""
datetime = tb.UInt64Col()
price = tb.UInt64Col()
vol = tb.UInt64Col()
2018-11-28 01:49:17 +08:00
#------------------------------------------------------------------------------
# K线数据
#------------------------------------------------------------------------------
2022-01-04 00:25:55 +08:00
2018-11-12 02:33:56 +08:00
def open_h5file(dest_dir, market, ktype):
filename = "{}/{}_{}.h5".format(dest_dir, market.lower(), ktype.lower())
2022-01-04 00:25:55 +08:00
h5file = tb.open_file(
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
)
2018-11-12 02:33:56 +08:00
return h5file
2018-11-25 16:46:07 +08:00
2022-01-04 00:25:55 +08:00
@hku_catch(None, trace=True)
2018-11-12 02:33:56 +08:00
def get_h5table(h5file, market, code):
try:
group = h5file.get_node("/", "data")
except:
group = h5file.create_group("/", "data")
tablename = market.upper() + code
try:
table = h5file.get_node(group, tablename)
except:
table = h5file.create_table(group, tablename, H5Record)
return table
def update_hdf5_extern_data(h5file, tablename, data_type):
2018-11-28 01:49:17 +08:00
"""更新周线、月线、15分钟线等扩展数据索引"""
2018-11-12 02:33:56 +08:00
def getWeekDate(olddate):
y = olddate // 100000000
m = olddate // 1000000 - y * 100
d = olddate // 10000 - (y * 10000 + m * 100)
tempdate = datetime.date(y, m, d)
# python中周一是第0天周五的第4天
tempweekdate = tempdate + datetime.timedelta(4 - tempdate.weekday())
newdate = tempweekdate.year * 100000000 + tempweekdate.month * 1000000 + tempweekdate.day * 10000
return newdate
def getMonthDate(olddate):
y = olddate // 100000000
m = olddate // 1000000 - y * 100
import calendar
_, d = calendar.monthrange(y, m)
return (y * 100000000 + m * 1000000 + d * 10000)
def getQuarterDate(olddate):
quarterDict = {1: 3, 2: 3, 3: 3, 4: 6, 5: 6, 6: 6, 7: 9, 8: 9, 9: 9, 10: 12, 11: 12, 12: 12}
d_dict = {3: 310000, 6: 300000, 9: 300000, 12: 310000}
y = olddate // 100000000
m = olddate // 1000000 - y * 100
new_m = quarterDict[m]
return (y * 100000000 + new_m * 1000000 + d_dict[new_m])
def getHalfyearDate(olddate):
y = olddate // 100000000
m = olddate // 1000000 - y * 100
return y * 100000000 + (6300000 if m < 7 else 12310000)
def getYearDate(olddate):
y = olddate // 100000000
return (y * 100000000 + 12310000)
def getMin60Date(olddate):
mint = olddate - olddate // 10000 * 10000
if mint <= 1030:
newdate = olddate // 10000 * 10000 + 1030
elif mint <= 1130:
newdate = olddate // 10000 * 10000 + 1130
elif mint <= 1400:
newdate = olddate // 10000 * 10000 + 1400
else:
newdate = olddate // 10000 * 10000 + 1500
return newdate
def getHour2Date(olddate):
mint = olddate - olddate // 10000 * 10000
if mint <= 1130:
newdate = olddate // 10000 * 10000 + 1130
else:
newdate = olddate // 10000 * 10000 + 1500
2018-11-12 02:33:56 +08:00
return newdate
2018-11-12 02:33:56 +08:00
def getMin15Date(olddate):
mint = olddate - olddate // 10000 * 10000
if mint <= 945:
newdate = olddate // 10000 * 10000 + 945
elif mint <= 1000:
newdate = olddate // 10000 * 10000 + 1000
elif mint <= 1015:
newdate = olddate // 10000 * 10000 + 1015
elif mint <= 1030:
newdate = olddate // 10000 * 10000 + 1030
elif mint <= 1045:
newdate = olddate // 10000 * 10000 + 1045
elif mint <= 1100:
newdate = olddate // 10000 * 10000 + 1100
elif mint <= 1115:
newdate = olddate // 10000 * 10000 + 1115
elif mint <= 1130:
newdate = olddate // 10000 * 10000 + 1130
elif mint <= 1315:
newdate = olddate // 10000 * 10000 + 1315
elif mint <= 1330:
newdate = olddate // 10000 * 10000 + 1330
elif mint <= 1345:
newdate = olddate // 10000 * 10000 + 1345
elif mint <= 1400:
newdate = olddate // 10000 * 10000 + 1400
elif mint <= 1415:
newdate = olddate // 10000 * 10000 + 1415
elif mint <= 1430:
newdate = olddate // 10000 * 10000 + 1430
elif mint <= 1445:
newdate = olddate // 10000 * 10000 + 1445
else:
newdate = olddate // 10000 * 10000 + 1500
return newdate
def getMin30Date(olddate):
mint = olddate - olddate // 10000 * 10000
if mint <= 1000:
newdate = olddate // 10000 * 10000 + 1000
elif mint <= 1030:
newdate = olddate // 10000 * 10000 + 1030
elif mint <= 1100:
newdate = olddate // 10000 * 10000 + 1100
elif mint <= 1130:
newdate = olddate // 10000 * 10000 + 1130
elif mint <= 1330:
newdate = olddate // 10000 * 10000 + 1330
elif mint <= 1400:
newdate = olddate // 10000 * 10000 + 1400
elif mint <= 1430:
newdate = olddate // 10000 * 10000 + 1430
else:
newdate = olddate // 10000 * 10000 + 1500
return newdate
def getNewDate(index_type, olddate):
if index_type == 'week':
return getWeekDate(olddate)
elif index_type == 'month':
return getMonthDate(olddate)
elif index_type == 'quarter':
return getQuarterDate(olddate)
elif index_type == 'halfyear':
return getHalfyearDate(olddate)
elif index_type == 'year':
return getYearDate(olddate)
elif index_type == 'min15':
return getMin15Date(olddate)
elif index_type == 'min30':
return getMin30Date(olddate)
elif index_type == 'min60':
return getMin60Date(olddate)
elif index_type == 'hour2':
return getHour2Date(olddate)
2018-11-12 02:33:56 +08:00
else:
return None
if data_type == 'DAY':
index_list = ('week', 'month', 'quarter', 'halfyear', 'year')
else:
index_list = ('min15', 'min30', 'min60', 'hour2')
2018-11-12 02:33:56 +08:00
groupDict = {}
for index_type in index_list:
try:
groupDict[index_type] = h5file.get_node("/", index_type)
except:
groupDict[index_type] = h5file.create_group("/", index_type)
try:
table = h5file.get_node("/data", tablename)
2022-01-04 00:25:55 +08:00
except Exception as e:
hku_error("{}".format(e))
2018-11-12 02:33:56 +08:00
return
for index_type in index_list:
hku_debug("{} update {} index".format(tablename, index_type))
2018-11-12 02:33:56 +08:00
try:
index_table = h5file.get_node(groupDict[index_type], tablename)
except:
index_table = h5file.create_table(groupDict[index_type], tablename, H5Index)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(index_type, int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(index_type, date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(index_type, date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
2018-11-25 16:46:07 +08:00
#------------------------------------------------------------------------------
2018-11-28 01:49:17 +08:00
# 分笔数据
2018-11-25 16:46:07 +08:00
#------------------------------------------------------------------------------
2022-01-04 00:25:55 +08:00
2018-11-25 16:46:07 +08:00
def open_trans_file(dest_dir, market):
filename = "{}/{}_trans.h5".format(dest_dir, market.lower())
2022-01-04 00:25:55 +08:00
h5file = tb.open_file(
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
)
2018-11-25 16:46:07 +08:00
return h5file
2022-01-04 00:25:55 +08:00
@hku_catch(trace=True)
2018-11-25 16:46:07 +08:00
def get_trans_table(h5file, market, code):
try:
group = h5file.get_node("/", "data")
except:
group = h5file.create_group("/", "data")
tablename = market.upper() + code
try:
table = h5file.get_node(group, tablename)
except:
table = h5file.create_table(group, tablename, H5Transaction)
return table
def update_hdf5_trans_index(h5file, tablename):
2018-11-28 01:49:17 +08:00
"""更新分笔数据按日索引"""
hku_debug("{} update trans index".format(tablename))
2018-11-25 16:46:07 +08:00
try:
table = h5file.get_node("/data", tablename)
except:
return
total = table.nrows
if 0 == total:
return
try:
group = h5file.get_node("/", 'index')
except:
group = h5file.create_group("/", 'index')
try:
index_table = h5file.get_node(group, tablename)
except:
index_table = h5file.create_table(group, tablename, H5Index)
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
2022-01-04 00:25:55 +08:00
last_date = int(table[-1]['datetime'] // 1000000 * 10000)
2018-11-25 16:46:07 +08:00
if index_last_date == last_date:
return
startix = int(index_table[-1]['start'])
pre_index_date = index_last_date
2018-11-25 16:46:07 +08:00
else:
startix = 0
date = int(table[0]['datetime'] // 1000000 * 10000)
2018-11-25 16:46:07 +08:00
pre_index_date = date
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
index = startix
for row in table[startix:]:
date = int(row['datetime'] // 1000000 * 10000)
2018-11-25 16:46:07 +08:00
cur_index_date = date
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
2022-01-04 00:25:55 +08:00
2018-11-25 16:46:07 +08:00
#------------------------------------------------------------------------------
2018-11-28 01:49:17 +08:00
# 分时数据
2018-11-25 16:46:07 +08:00
#------------------------------------------------------------------------------
def open_time_file(dest_dir, market):
filename = "{}/{}_time.h5".format(dest_dir, market.lower())
2022-01-04 00:25:55 +08:00
h5file = tb.open_file(
filename, "a", filters=tb.Filters(complevel=HDF5_COMPRESS_LEVEL, complib='zlib', shuffle=True)
)
2018-11-25 16:46:07 +08:00
return h5file
2022-01-04 00:25:55 +08:00
@hku_catch(None, trace=True)
2018-11-25 16:46:07 +08:00
def get_time_table(h5file, market, code):
try:
group = h5file.get_node("/", "data")
except:
group = h5file.create_group("/", "data")
tablename = market.upper() + code
try:
table = h5file.get_node(group, tablename)
except:
table = h5file.create_table(group, tablename, H5MinuteTime)
return table