test/hikyuu2
1
0
Fork 0
mirror of https://gitee.com/fasiondog/hikyuu.git synced 2024-12-02 20:08:26 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

完善基础设施及界面

Browse Source
This commit is contained in:
fasiondog 2020-12-05 17:53:38 +08:00
parent e5ec21ac66
commit 36a08676d4
11 changed files with 244 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hikyuu/fetcher/proxy/__init__.py Normal file
View File

@ -0,0 +1,12 @@
# coding:utf-8
#
# The MIT License (MIT)
#
# Created on: 2020-11-29
# Author: fasiondog
__version__ = "0.0.1"
from .proxy import request_with_proxy, request_with_local
__all__ = ['request_with_proxy', 'request_with_local']

34
hikyuu/fetcher/proxy/proxy.py Normal file
View File

@ -0,0 +1,34 @@
# coding:utf-8
#
# The MIT License (MIT)
#
# Created on: 2020-11-29
# Author: fasiondog
import requests
import json
import datetime
from hikyuu.util import hku_logger, hku_check
from .zhima import get_proxy
def request_with_proxy(url):
"""通过代理进行请求，访问失败将抛出异常"""
# 获取到的 ip 可能无法访问相应的 url，重试10次，以便找到能用的 proxy
new = False
for i in range(10): # pylint: disable=unused-variable
try:
proxy = get_proxy(new)
hku_check(proxy, "Failed get proxy!")
proxies = {'http': 'http://{}'.format(proxy)}
hku_logger.info("use proxy: {}".format(proxies['http']))
return requests.get(url, proxies=proxies).text
except:
new = True
raise Exception("无法通过代理访问！")
def request_with_local(url):
"""通过本机ip直接获取请求，访问失败将抛出异常"""
return requests.get(url).text

48
hikyuu/fetcher/proxy.py → hikyuu/fetcher/proxy/zhima.py
View File

@ -14,9 +14,11 @@ from hikyuu.util import hku_logger, hku_catch, hku_check
zhimahttp_url = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=0&city=0&yys=0&port=1&time=1&ts=1&ys=1&cs=1&lb=1&sb=0&pb=45&mr=2&regions=110000,130000,140000,310000,320000,330000,370000,410000'
@hku_catch(retry=10)
@hku_catch(retry=3)
def request_proxy_from_zhima():
"""返回格式如下：
"""从芝麻申请代理IP
返回格式如下：
{'ip': '115.226.137.200',
'port': 4223,
'expire_time': '2020-12-01 00:12:33'（实际为datetime类型）
@ -37,11 +39,11 @@ def request_proxy_from_zhima():
g_last_proxy = None
def get_proxy_from_zhima(new=False):
"""从芝麻http代理获取代理
def get_proxy(new=False):
"""从芝麻获取代理
:param boolean new: True 强制重新申请 | False 尽可能使用上次获取的代理ip
:return: 'host ip:port'， 如: '183.164.239.57:4264'
:return: None | 'host ip:port'， 如: '183.164.239.57:4264'
"""
global g_last_proxy
if new or g_last_proxy is None:
@ -51,39 +53,3 @@ def get_proxy_from_zhima(new=False):
if current_time >= g_last_proxy['expire_time']: # 简单不严格的时间判断，两者时间可能不同步
g_last_proxy = request_proxy_from_zhima()
return '{}:{}'.format(g_last_proxy['ip'], g_last_proxy['port']) if g_last_proxy else None
def get_proxy(new=False):
"""
返回代理地址，格式 :代理ip地址:代理端口号，如：183.164.239.57:4264
如无法获取时，返回 None
:param boolean new: True 强制重新申请 | False 尽可能使用上次获取的代理ip
:return: 'host ip:port'， 如: '183.164.239.57:4264'
"""
return get_proxy_from_zhima(new)
#return None
def request_with_proxy(url):
"""通过代理进行请求"""
new = False
for i in range(10): # pylint: disable=unused-variable
try:
proxy = get_proxy(new)
if proxy is None:
# 因为get_proxy 已经进行过重试获取，这里直接返回
hku_logger.warning("Failed get proxy!")
return None
proxies = {'http': 'http://{}'.format(proxy)}
hku_logger.info("use proxy: {}".format(proxies['http']))
return requests.get(url, proxies=proxies).text
except:
new = True
return None
@hku_catch()
def request_with_local(url):
"""通过本机ip直接获取请求"""
return requests.get(url).text

5
hikyuu/fetcher/stock/zh_stock_a_sina_qq.py
View File

@ -119,10 +119,8 @@ def parse_one_result_qq(resultstr):
def request_data(querystr, parse_one_result, use_proxy=False):
"""请求失败将抛出异常"""
query = request_with_proxy(querystr) if use_proxy else request_with_local(querystr)
if query is None:
hku_logger.error('请求失败！无法获取数据')
return
query = query.split('\n')
result = []
for tmpstr in query:
@ -134,6 +132,7 @@ def request_data(querystr, parse_one_result, use_proxy=False):
def get_spot(stocklist, source='sina', use_proxy=False):
"""获取失败时，请抛出异常"""
if source == 'sina':
queryStr = "http://hq.sinajs.cn/list="
max_size = 140

33
hikyuu/gui/HikyuuTDX.py
View File

@ -22,7 +22,7 @@ from hikyuu.gui.data.UsePytdxImportToH5Thread import UsePytdxImportToH5Thread
from hikyuu.gui.data.CollectThread import CollectThread
from hikyuu.data import hku_config_template
from hikyuu.util.mylog import add_class_logger_handler, class_logger
from hikyuu.util.mylog import add_class_logger_handler, class_logger, hku_logger
class EmittingStream(QObject):
@ -108,11 +108,18 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
def normalOutputWritten(self, text):
"""普通打印信息重定向"""
if text.find('[WARN]') >= 0:
text = '<font color="#0000FF">{}</font>'.format(text)
elif text.find('[ERROR]') >= 0:
text = '<font color="#FF0000">{}</font>'.format(text)
elif text.find('[CRITICAL]') >= 0:
text = '<span style="background-color: #ff0000;">{}</span>'.format(text)
else:
# 主动加入<font>标签，避免 append 时多加入空行
text = '<font color="#000000">{}</font>'.format(text)
cursor = self.log_textEdit.textCursor()
cursor.movePosition(QTextCursor.End)
cursor.insertText(text)
self.log_textEdit.setTextCursor(cursor)
self.log_textEdit.ensureCursorVisible()
self.log_textEdit.append(text)
def initLogger(self):
if not self._capture_output:
@ -128,12 +135,14 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
con, [MyMainWindow, CollectThread, UsePytdxImportToH5Thread, UseTdxImportToH5Thread],
logging.INFO
)
hku_logger.addHandler(con)
def initUI(self):
if self._capture_output:
stream = EmittingStream(textWritten=self.normalOutputWritten)
sys.stdout = stream
sys.stderr = stream
self.log_textEdit.document().setMaximumBlockCount(500)
current_dir = os.path.dirname(__file__)
self.setWindowIcon(QIcon("{}/hikyuu.ico".format(current_dir)))
@ -518,13 +527,13 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
def stop_collect(self):
self.logger.info("终止采集！")
if self.collect_sh_thread is not None:
self.collect_sh_thread.working = False
self.collect_sh_thread.stop()
self.collect_sh_thread.terminate()
del self.collect_sh_thread
self.collect_sh_thread = None
if self.collect_sz_thread is not None:
self.collect_sz_thread.working = False
self.collect_sz_thread.stop()
self.collect_sz_thread.terminate()
del self.collect_sz_thread
self.collect_sz_thread = None
@ -532,19 +541,27 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
@pyqtSlot()
def on_collect_start_pushButton_clicked(self):
if self.collect_running:
self.collect_start_pushButton.setEnabled(False)
self.collect_status_Label.setText("正在停止...")
QApplication.processEvents()
self.stop_collect()
self.collect_status_Label.setText("已停止")
self.collect_start_pushButton.setText("启动定时采集")
self.collect_running = False
self.collect_status_Label.setText("已停止")
self.collect_start_pushButton.setEnabled(True)
else:
config = self.getCurrentConfig()
if not config.getboolean("mysql", "enable", fallback=False):
QMessageBox.critical(self, "定时采集", "仅在存储设置为 MySQL 时支持定时采集！")
return
self.collect_status_Label.setText("运行中...")
self.collect_status_Label.setText("正在启动...")
self.collect_start_pushButton.setEnabled(False)
QApplication.processEvents()
self.start_collect()
self.collect_start_pushButton.setText("停止采集")
self.collect_running = True
self.collect_status_Label.setText("运行中...")
self.collect_start_pushButton.setEnabled(True)
class_logger(MyMainWindow)

54
hikyuu/gui/data/CollectThread.py
View File

@ -8,7 +8,7 @@ from PyQt5.QtCore import QThread
import mysql.connector
from mysql.connector import errorcode
from hikyuu.util.mylog import class_logger
from hikyuu.util import *
from hikyuu.data.common_mysql import get_stock_list
from hikyuu.fetcher.stock.zh_stock_a_sina_qq import get_spot
@ -28,6 +28,19 @@ class CollectThread(QThread):
'port': config['mysql']['port']
}
self._connect = None
self.quotations = []
if config['quotation']['stock']:
self.quotations.append('stock')
if config['quotation']['fund']:
self.quotations.append('fund')
self.logger.info(self.quotations)
def stop(self):
self.working = False
if self._connect is not None:
hku_info('关闭数据库连接', self.logger)
self._connect.close()
self.wait()
def __del__(self):
self.working = False
@ -37,9 +50,11 @@ class CollectThread(QThread):
def run(self):
self.logger.info("{} 数据采集同步线程启动 ({})".format(self.market, self.currentThreadId()))
stk_list = self.get_stock_list()
hku_warn_if(not stk_list, "从数据库中获取的股票列表为空！", self.logger)
while self.working == True:
start = time.time()
self.collect()
self.collect(stk_list)
end = time.time()
x = end - start
if x < self._interval:
@ -48,23 +63,32 @@ class CollectThread(QThread):
self.sleep(delta)
self.logger.info("{} 数据采集同步线程终止 ({})!".format(self.market, self.currentThreadId()))
def collect(self):
@hku_catch()
def collect(self, stk_list):
self.logger.info("{} collect".format(self.market))
self.logger.info("{}".format(len(self.get_stock_list())))
def get_connect(self):
if self._connect is None:
try:
self._connect = mysql.connector.connect(**self._db_config)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
self.logger.error("MYSQL密码或用户名错误！")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
self.logger.error("MySQL数据库不存在！")
else:
self.logger.error("连接数据库失败，{}".format(err.msg))
except:
self.logger.error("未知原因导致无法连接数据库！")
if self._connect:
return self._connect
try:
self._connect = mysql.connector.connect(**self._db_config)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
self.logger.error("MYSQL密码或用户名错误！")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
self.logger.error("MySQL数据库不存在！")
else:
self.logger.error("连接数据库失败，{}".format(err.msg))
except:
self.logger.error("未知原因导致无法连接数据库！")
return self._connect
@hku_catch(retry=2, ret=[])
def get_stock_list(self):
connect = self.get_connect()
stk_list = get_stock_list(connect, self.market.upper(), self.quotations)
return ["{}{}".format(self.market.lower(), item[2]) for item in stk_list if item[3] == 1]
class_logger(CollectThread)

2
hikyuu/gui/data/MainWindow.py
View File

@ -388,7 +388,7 @@ class Ui_MainWindow(object):
self.log_textEdit = QtWidgets.QTextEdit(self.tab_5)
self.log_textEdit.setGeometry(QtCore.QRect(10, 10, 571, 511))
self.log_textEdit.setReadOnly(True)
self.log_textEdit.setAcceptRichText(False)
self.log_textEdit.setAcceptRichText(True)
self.log_textEdit.setObjectName("log_textEdit")
self.tabWidget.addTab(self.tab_5, "")
MainWindow.setCentralWidget(self.centralwidget)

2
hikyuu/gui/data/MainWindow.ui
View File

@ -886,7 +886,7 @@ p, li { white-space: pre-wrap; }
<bool>true</bool>
</property>
<property name="acceptRichText">
<bool>false</bool>
<bool>true</bool>
</property>
</widget>
</widget>

14
hikyuu/util/__init__.py
View File

@ -22,4 +22,18 @@ __all__ = [
'hku_check_throw',
'hku_check_ignore',
'hku_catch',
'hku_trace',
'hku_debug',
'hku_info',
'hku_warn',
'hku_error',
'hku_fatal',
'hku_trace_if',
'hku_debug_if',
'hku_info_if',
'hku_warn_if',
'hku_info_if',
'hku_warn_if',
'hku_error_if',
'hku_fatal_if',
]

23
hikyuu/util/check.py
View File

@ -47,7 +47,10 @@ def checkif(expression, message, excepion=None, **kwargs):
def hku_check(exp, msg):
if not exp:
raise HKUCheckError(exp, msg)
st = traceback.extract_stack()[-2]
check_exp = st._line.split(',')[0]
errmsg = "{}) {} [{}] [{}:{}]".format(check_exp, msg, st.name, st.filename, st.lineno)
raise HKUCheckError(exp, errmsg)
def hku_check_throw(expression, message, excepion=None, **kwargs):
@ -58,16 +61,22 @@ def hku_check_throw(expression, message, excepion=None, **kwargs):
:param Exception exception: 指定的异常类，为None时，为默认 HKUCheckError 异常
"""
if not expression:
st = traceback.extract_stack()[-2]
check_exp = st._line.split(',')[0]
errmsg = "{}) {} [{}] [{}:{}]".format(check_exp, message, st.name, st.filename, st.lineno)
if excepion is None:
raise HKUCheckError(expression, message)
raise HKUCheckError(expression, errmsg)
else:
raise excepion(message, **kwargs)
raise excepion(errmsg, **kwargs)
def hku_check_ignore(exp, msg=None):
"""可忽略的检查"""
if not exp:
raise HKUIngoreError(exp, msg)
st = traceback.extract_stack()[-2]
check_exp = st._line.split(',')[0]
errmsg = "{}) {} [{}] [{}:{}]".format(check_exp, msg, st.name, st.filename, st.lineno)
raise HKUIngoreError(exp, errmsg)
def get_exception_info():
@ -95,7 +104,8 @@ def hku_catch(ret=None, trace=False, callback=None, retry=1):
hku_logger.error(
"{} [{}.{}]".format(get_exception_info(), func.__module__, func.__name__)
)
traceback.print_exc()
if trace:
traceback.print_exc()
if callback and i == (retry - 1):
callback(*args, **kargs)
except:
@ -104,7 +114,8 @@ def hku_catch(ret=None, trace=False, callback=None, retry=1):
get_exception_info(), func.__module__, func.__name__
)
)
traceback.print_exc()
if trace:
traceback.print_exc()
if callback and i == (retry - 1):
callback(*args, **kargs)
return ret

93
hikyuu/util/mylog.py
View File

@ -3,7 +3,7 @@
# cp936
import logging
import sys
import traceback
import time
from functools import wraps
@ -53,3 +53,94 @@ def add_class_logger_handler(handler, class_list, level=logging.INFO):
logger = logging.getLogger("{}".format(cls.__name__))
logger.addHandler(handler)
logger.setLevel(level)
def hku_debug(msg, logger=None):
st = traceback.extract_stack()[-2]
if logger:
logger.debug("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.debug("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
hku_trace = hku_debug
def hku_info(msg, logger=None):
st = traceback.extract_stack()[-2]
if logger:
logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_warn(msg, logger=None):
st = traceback.extract_stack()[-2]
if logger:
logger.warning("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.warning("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_error(msg, logger=None):
st = traceback.extract_stack()[-2]
if logger:
logger.error("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.error("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_fatal(msg, logger=None):
st = traceback.extract_stack()[-2]
if logger:
logger.critical("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.critical("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_debug_if(exp, msg, logger=None):
if exp:
st = traceback.extract_stack()[-2]
if logger:
logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
hku_trace_if = hku_debug_if
def hku_info_if(exp, msg, logger=None):
if exp:
st = traceback.extract_stack()[-2]
if logger:
logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.info("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_warn_if(exp, msg, logger=None):
if exp:
st = traceback.extract_stack()[-2]
if logger:
logger.warning("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.warning("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_error_if(exp, msg, logger=None):
if exp:
st = traceback.extract_stack()[-2]
if logger:
logger.error("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.error("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
def hku_fatal_if(exp, msg, logger=None):
if exp:
st = traceback.extract_stack()[-2]
if logger:
logger.critical("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))
else:
hku_logger.critical("{} [{}] ({}:{})".format(msg, st.name, st.filename, st.lineno))