定时采集(continue)

This commit is contained in:
fasiondog 2020-11-29 00:13:40 +08:00
parent da79950a4d
commit 62cab0619d
11 changed files with 294 additions and 79 deletions

View File

@ -8,8 +8,8 @@ import datetime
from configparser import ConfigParser
from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QMessageBox
from PyQt5.QtCore import pyqtSlot
from PyQt5.QtGui import QIcon
from PyQt5.QtCore import pyqtSlot, QObject, pyqtSignal
from PyQt5.QtGui import QIcon, QTextCursor
import mysql.connector
from mysql.connector import errorcode
@ -19,18 +19,35 @@ from hikyuu.gui.data.MainWindow import *
from hikyuu.gui.data.EscapetimeThread import EscapetimeThread
from hikyuu.gui.data.UseTdxImportToH5Thread import UseTdxImportToH5Thread
from hikyuu.gui.data.UsePytdxImportToH5Thread import UsePytdxImportToH5Thread
from hikyuu.gui.data.CollectThread import CollectThread
from hikyuu.data import hku_config_template
class EmittingStream(QObject):
"""输出重定向至QT"""
textWritten = pyqtSignal(str)
def write(self, text):
self.textWritten.emit(str(text))
def flush(self):
pass
class MyMainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
def __init__(self, parent=None, capture_output=False):
super(MyMainWindow, self).__init__(parent)
self._capture_output = capture_output #Python stdout
self.logger = logging.getLogger(self.__class__.__name__)
self.setupUi(self)
self.initUI()
self.initLogger()
self.initThreads()
self.logger.info("Init...")
def closeEvent(self, event):
self.stop_collect()
if self.import_running:
QMessageBox.about(self, '', '')
event.ignore()
@ -89,7 +106,39 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
shutil.copytree(dirname, data_dir + '/block')
os.remove(data_dir + '/block/__init__.py')
def normalOutputWritten(self, text):
"""普通打印信息重定向"""
cursor = self.log_textEdit.textCursor()
cursor.movePosition(QTextCursor.End)
cursor.insertText(text)
self.log_textEdit.setTextCursor(cursor)
self.log_textEdit.ensureCursorVisible()
def initLogger(self):
if not self._capture_output:
return
#普通日志输出控制台
con = logging.StreamHandler(EmittingStream(textWritten=self.normalOutputWritten))
FORMAT = logging.Formatter(
'%(asctime)-15s [%(levelname)s]: %(message)s [%(name)s::%(funcName)s]'
)
con.setFormatter(FORMAT)
logger_name_list = [
self.__class__.__name__, CollectThread.__name__, UsePytdxImportToH5Thread.__name__,
UseTdxImportToH5Thread.__name__
]
for name in logger_name_list:
logger = logging.getLogger(name)
logger.addHandler(con)
logger.setLevel(logging.INFO)
def initUI(self):
if self._capture_output:
stream = EmittingStream(textWritten=self.normalOutputWritten)
sys.stdout = stream
sys.stderr = stream
current_dir = os.path.dirname(__file__)
self.setWindowIcon(QIcon("{}/hikyuu.ico".format(current_dir)))
self.setFixedSize(self.width(), self.height())
@ -107,6 +156,8 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.time_start_dateEdit.setDate(today - datetime.timedelta(7))
self.trans_start_dateEdit.setMinimumDate(today - datetime.timedelta(90))
self.time_start_dateEdit.setMinimumDate(today - datetime.timedelta(300))
self.collect_running = False
self.collect_status_Label.setText("未启动")
#读取保存的配置文件信息,如果不存在,则使用默认配置
this_dir = self.getUserConfigDir()
@ -244,6 +295,8 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.escape_time_thread = None
self.hdf5_import_thread = None
self.mysql_import_thread = None
self.collect_sh_thread = None
self.collect_sz_thread = None
self.import_running = False
self.hdf5_import_progress_bar = {
@ -460,6 +513,43 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
self.escape_time_thread.message.connect(self.on_message_from_thread)
self.escape_time_thread.start()
def start_collect(self):
self.collect_sh_thread = CollectThread(self.getCurrentConfig(), 'SH', 2)
self.collect_sh_thread.start()
self.collect_sz_thread = CollectThread(self.getCurrentConfig(), 'SZ', 2)
self.collect_sz_thread.start()
def stop_collect(self):
self.logger.info("终止采集!")
if self.collect_sh_thread is not None:
self.collect_sh_thread.working = False
self.collect_sh_thread.terminate()
del self.collect_sh_thread
self.collect_sh_thread = None
if self.collect_sz_thread is not None:
self.collect_sz_thread.working = False
self.collect_sz_thread.terminate()
del self.collect_sz_thread
self.collect_sz_thread = None
@pyqtSlot()
def on_collect_start_pushButton_clicked(self):
if self.collect_running:
self.stop_collect()
self.collect_status_Label.setText("已停止")
self.collect_start_pushButton.setText("启动定时采集")
self.collect_running = False
else:
config = self.getCurrentConfig()
if not config.getboolean("mysql", "enable", fallback=False):
QMessageBox.critical(self, "定时采集", "仅在存储设置为 MySQL 时支持定时采集!")
return
self.collect_status_Label.setText("运行中...")
self.start_collect()
self.collect_start_pushButton.setText("停止采集")
self.collect_running = True
def start():
app = QApplication(sys.argv)
@ -471,16 +561,15 @@ def start():
if __name__ == "__main__":
app = QApplication(sys.argv)
if (len(sys.argv) > 1 and sys.argv[1] == '0'):
FORMAT = '%(asctime)-15s %(levelname)s: %(message)s [%(name)s::%(funcName)s]'
FORMAT = '%(asctime)-15s [%(levelname)s]: %(message)s [%(name)s::%(funcName)s]'
logging.basicConfig(
format=FORMAT, level=logging.INFO, handlers=[
logging.StreamHandler(),
]
)
capture_output = False
myWin = MyMainWindow(capture_output=False)
else:
capture_output = True
myWin = MyMainWindow(capture_output=True)
myWin = MyMainWindow(None)
myWin.show()
sys.exit(app.exec())

View File

@ -0,0 +1,64 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
# cp936
import logging
import time
from PyQt5.QtCore import QThread
import mysql.connector
from mysql.connector import errorcode
class CollectThread(QThread):
def __init__(self, config, market='SH', interval=60):
super(self.__class__, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.working = True
self._interval = interval
self._config = config
self.market = market
assert config.getboolean("mysql", "enable", fallback=False)
self._db_config = {
'user': config['mysql']['usr'],
'password': config['mysql']['pwd'],
'host': config['mysql']['host'],
'port': config['mysql']['port']
}
self._connect = None
def __del__(self):
self.working = False
if self._connect is not None:
self._connect.close()
self.wait()
def run(self):
self.logger.info("{} 数据采集同步线程启动 ({})".format(self.market, self.currentThreadId()))
while self.working == True:
start = time.time()
self.collect()
end = time.time()
x = end - start
if x < self._interval:
delta = int(self._interval - x)
self.logger.info("{} {} 秒后重新采集".format(self.market, delta))
self.sleep(delta)
self.logger.info("{} 数据采集同步线程终止 ({})!".format(self.market, self.currentThreadId()))
def collect(self):
self.logger.info("{} collect".format(self.market))
def get_connect(self):
if self._connect is None:
try:
self._connect = mysql.connector.connect(**self._db_config)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
self.logger.error("MYSQL密码或用户名错误")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
self.logger.error("MySQL数据库不存在")
else:
self.logger.error("连接数据库失败,{}".format(err.msg))
except:
self.logger.error("未知原因导致无法连接数据库!")
return self._connect

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
from pytdx.hq import TdxHq_API
from hikyuu.data.pytdx_to_h5 import import_time
@ -39,6 +40,7 @@ class ProgressBar:
class ImportPytdxTimeToH5:
def __init__(self, queue, sqlitefile, market, quotations, ip, port, dest_dir, max_days):
self.logger = logging.getLogger(self.__class__.__name__)
self.task_name = 'IMPORT_TIME'
self.queue = queue
self.sqlitefile = sqlitefile
@ -66,7 +68,7 @@ class ImportPytdxTimeToH5:
progress=progress
)
except Exception as e:
print(e)
self.logger.error(e)
finally:
connect.commit()
connect.close()

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
import mysql.connector
from pytdx.hq import TdxHq_API
@ -43,6 +44,7 @@ class ImportPytdxToH5:
def __init__(
self, queue, config, market, ktype, quotations, ip, port, dest_dir, start_datetime
):
self.logger = logging.getLogger(self.__class__.__name__)
self.task_name = 'IMPORT_KDATA'
self.queue = queue
self.config = config
@ -59,7 +61,7 @@ class ImportPytdxToH5:
sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
connect = sqlite3.connect(sqlite_file, timeout=1800)
import_data = h5_import_data
print('use hdf5 import kdata')
self.logger.debug('use hdf5 import kdata')
else:
db_config = {
'user': self.config['mysql']['usr'],
@ -69,7 +71,7 @@ class ImportPytdxToH5:
}
connect = mysql.connector.connect(**db_config)
import_data = mysql_import_data
print('use mysql import kdata')
self.logger.debug('use mysql import kdata')
count = 0
try:
@ -81,7 +83,7 @@ class ImportPytdxToH5:
self.startDatetime, progress
)
except Exception as e:
print("ImportPytdxToH5Task failed!", e)
self.logger.error("ImportPytdxToH5Task failed! {}".format(e))
#self.queue.put([self.task_name, self.market, self.ktype, str(e), count])
finally:
connect.commit()

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
from pytdx.hq import TdxHq_API
from hikyuu.data.pytdx_to_h5 import import_data, import_trans
@ -39,6 +40,7 @@ class ProgressBar:
class ImportPytdxTransToH5:
def __init__(self, queue, sqlitefile, market, quotations, ip, port, dest_dir, max_days):
self.logger = logging.getLogger(self.__class__.__name__)
self.task_name = 'IMPORT_TRANS'
self.queue = queue
self.sqlitefile = sqlitefile
@ -67,7 +69,7 @@ class ImportPytdxTransToH5:
progress=progress
)
except Exception as e:
print(e)
self.logger.error(e)
finally:
connect.commit()
connect.close()

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
from hikyuu.data.tdx_to_h5 import tdx_import_data
@ -40,6 +41,7 @@ class ProgressBar:
class ImportTdxToH5Task:
def __init__(self, queue, sqlitefile, market, ktype, quotations, src_dir, dest_dir):
super(self.__class__, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.task_name = 'IMPORT_KDATA'
self.queue = queue
self.sqlitefile = sqlitefile
@ -76,5 +78,5 @@ class ImportTdxToH5Task:
progress
)
except Exception as e:
print(e)
self.logger.error(e)
self.queue.put([self.task_name, self.market, self.ktype, None, count])

View File

@ -23,6 +23,7 @@
# SOFTWARE.
import os
import logging
import hashlib
import sqlite3
import urllib.request
@ -38,6 +39,7 @@ from hikyuu.data.pytdx_weight_to_mysql import pytdx_import_weight_to_mysql
class ImportWeightToSqliteTask:
def __init__(self, queue, config, dest_dir):
self.logger = logging.getLogger(self.__class__.__name__)
self.queue = queue
self.config = config
self.dest_dir = dest_dir
@ -50,7 +52,7 @@ class ImportWeightToSqliteTask:
sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
connect = sqlite3.connect(sqlite_file, timeout=1800)
pytdx_import_weight = pytdx_import_weight_to_sqlite
print('use sqlite import weight')
self.logger.debug('use sqlite import weight')
else:
db_config = {
'user': self.config['mysql']['usr'],
@ -60,7 +62,7 @@ class ImportWeightToSqliteTask:
}
connect = mysql.connector.connect(**db_config)
pytdx_import_weight = pytdx_import_weight_to_mysql
print('use mysql import weight')
self.logger.debug('use mysql import weight')
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
@ -127,6 +129,7 @@ class ImportWeightToSqliteTask:
api.disconnect()
except Exception as e:
self.logger.error(e)
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:

View File

@ -312,9 +312,9 @@ class Ui_MainWindow(object):
self.tabWidget.addTab(self.tab_2, "")
self.tab = QtWidgets.QWidget()
self.tab.setObjectName("tab")
self.time_no_week_checkBox = QtWidgets.QCheckBox(self.tab)
self.time_no_week_checkBox.setGeometry(QtCore.QRect(40, 200, 101, 16))
self.time_no_week_checkBox.setObjectName("time_no_week_checkBox")
self.collect_no_week_checkBox = QtWidgets.QCheckBox(self.tab)
self.collect_no_week_checkBox.setGeometry(QtCore.QRect(40, 200, 101, 16))
self.collect_no_week_checkBox.setObjectName("collect_no_week_checkBox")
self.layoutWidget3 = QtWidgets.QWidget(self.tab)
self.layoutWidget3.setGeometry(QtCore.QRect(40, 160, 158, 22))
self.layoutWidget3.setObjectName("layoutWidget3")
@ -324,62 +324,62 @@ class Ui_MainWindow(object):
self.label_23 = QtWidgets.QLabel(self.layoutWidget3)
self.label_23.setObjectName("label_23")
self.horizontalLayout_10.addWidget(self.label_23)
self.time_sample_spinBox = QtWidgets.QSpinBox(self.layoutWidget3)
self.time_sample_spinBox.setMaximum(86400)
self.time_sample_spinBox.setObjectName("time_sample_spinBox")
self.horizontalLayout_10.addWidget(self.time_sample_spinBox)
self.widget = QtWidgets.QWidget(self.tab)
self.widget.setGeometry(QtCore.QRect(40, 30, 201, 25))
self.widget.setObjectName("widget")
self.horizontalLayout_11 = QtWidgets.QHBoxLayout(self.widget)
self.collect_sample_spinBox = QtWidgets.QSpinBox(self.layoutWidget3)
self.collect_sample_spinBox.setMaximum(86400)
self.collect_sample_spinBox.setObjectName("collect_sample_spinBox")
self.horizontalLayout_10.addWidget(self.collect_sample_spinBox)
self.layoutWidget4 = QtWidgets.QWidget(self.tab)
self.layoutWidget4.setGeometry(QtCore.QRect(40, 30, 201, 25))
self.layoutWidget4.setObjectName("layoutWidget4")
self.horizontalLayout_11 = QtWidgets.QHBoxLayout(self.layoutWidget4)
self.horizontalLayout_11.setContentsMargins(0, 0, 0, 0)
self.horizontalLayout_11.setObjectName("horizontalLayout_11")
self.time_start_pushButton = QtWidgets.QPushButton(self.widget)
self.time_start_pushButton.setObjectName("time_start_pushButton")
self.horizontalLayout_11.addWidget(self.time_start_pushButton)
self.time_status = QtWidgets.QLabel(self.widget)
self.time_status.setObjectName("time_status")
self.horizontalLayout_11.addWidget(self.time_status)
self.widget1 = QtWidgets.QWidget(self.tab)
self.widget1.setGeometry(QtCore.QRect(40, 80, 216, 56))
self.widget1.setObjectName("widget1")
self.verticalLayout = QtWidgets.QVBoxLayout(self.widget1)
self.collect_start_pushButton = QtWidgets.QPushButton(self.layoutWidget4)
self.collect_start_pushButton.setObjectName("collect_start_pushButton")
self.horizontalLayout_11.addWidget(self.collect_start_pushButton)
self.collect_status_Label = QtWidgets.QLabel(self.layoutWidget4)
self.collect_status_Label.setObjectName("collect_status_Label")
self.horizontalLayout_11.addWidget(self.collect_status_Label)
self.layoutWidget5 = QtWidgets.QWidget(self.tab)
self.layoutWidget5.setGeometry(QtCore.QRect(40, 80, 216, 56))
self.layoutWidget5.setObjectName("layoutWidget5")
self.verticalLayout = QtWidgets.QVBoxLayout(self.layoutWidget5)
self.verticalLayout.setContentsMargins(0, 0, 0, 0)
self.verticalLayout.setObjectName("verticalLayout")
self.horizontalLayout_8 = QtWidgets.QHBoxLayout()
self.horizontalLayout_8.setObjectName("horizontalLayout_8")
self.label_21 = QtWidgets.QLabel(self.widget1)
self.label_21 = QtWidgets.QLabel(self.layoutWidget5)
self.label_21.setObjectName("label_21")
self.horizontalLayout_8.addWidget(self.label_21)
self.horizontalLayout_6 = QtWidgets.QHBoxLayout()
self.horizontalLayout_6.setObjectName("horizontalLayout_6")
self.time_phase1_start_timeEdit = QtWidgets.QTimeEdit(self.widget1)
self.time_phase1_start_timeEdit.setObjectName("time_phase1_start_timeEdit")
self.horizontalLayout_6.addWidget(self.time_phase1_start_timeEdit)
self.label_22 = QtWidgets.QLabel(self.widget1)
self.collect_phase1_start_timeEdit = QtWidgets.QTimeEdit(self.layoutWidget5)
self.collect_phase1_start_timeEdit.setObjectName("collect_phase1_start_timeEdit")
self.horizontalLayout_6.addWidget(self.collect_phase1_start_timeEdit)
self.label_22 = QtWidgets.QLabel(self.layoutWidget5)
self.label_22.setObjectName("label_22")
self.horizontalLayout_6.addWidget(self.label_22)
self.time_phase1_last_timeEdit = QtWidgets.QTimeEdit(self.widget1)
self.time_phase1_last_timeEdit.setObjectName("time_phase1_last_timeEdit")
self.horizontalLayout_6.addWidget(self.time_phase1_last_timeEdit)
self.collect_phase1_last_timeEdit = QtWidgets.QTimeEdit(self.layoutWidget5)
self.collect_phase1_last_timeEdit.setObjectName("collect_phase1_last_timeEdit")
self.horizontalLayout_6.addWidget(self.collect_phase1_last_timeEdit)
self.horizontalLayout_8.addLayout(self.horizontalLayout_6)
self.verticalLayout.addLayout(self.horizontalLayout_8)
self.horizontalLayout_9 = QtWidgets.QHBoxLayout()
self.horizontalLayout_9.setObjectName("horizontalLayout_9")
self.label_24 = QtWidgets.QLabel(self.widget1)
self.label_24 = QtWidgets.QLabel(self.layoutWidget5)
self.label_24.setObjectName("label_24")
self.horizontalLayout_9.addWidget(self.label_24)
self.horizontalLayout_7 = QtWidgets.QHBoxLayout()
self.horizontalLayout_7.setObjectName("horizontalLayout_7")
self.time_phase2_start_timeEdit = QtWidgets.QTimeEdit(self.widget1)
self.time_phase2_start_timeEdit.setObjectName("time_phase2_start_timeEdit")
self.horizontalLayout_7.addWidget(self.time_phase2_start_timeEdit)
self.label_25 = QtWidgets.QLabel(self.widget1)
self.collect_phase2_start_timeEdit = QtWidgets.QTimeEdit(self.layoutWidget5)
self.collect_phase2_start_timeEdit.setObjectName("collect_phase2_start_timeEdit")
self.horizontalLayout_7.addWidget(self.collect_phase2_start_timeEdit)
self.label_25 = QtWidgets.QLabel(self.layoutWidget5)
self.label_25.setObjectName("label_25")
self.horizontalLayout_7.addWidget(self.label_25)
self.time_phase2_last_timeEdit = QtWidgets.QTimeEdit(self.widget1)
self.time_phase2_last_timeEdit.setObjectName("time_phase2_last_timeEdit")
self.horizontalLayout_7.addWidget(self.time_phase2_last_timeEdit)
self.collect_phase2_last_timeEdit = QtWidgets.QTimeEdit(self.layoutWidget5)
self.collect_phase2_last_timeEdit.setObjectName("collect_phase2_last_timeEdit")
self.horizontalLayout_7.addWidget(self.collect_phase2_last_timeEdit)
self.horizontalLayout_9.addLayout(self.horizontalLayout_7)
self.verticalLayout.addLayout(self.horizontalLayout_9)
self.tabWidget.addTab(self.tab, "")
@ -387,6 +387,8 @@ class Ui_MainWindow(object):
self.tab_5.setObjectName("tab_5")
self.log_textEdit = QtWidgets.QTextEdit(self.tab_5)
self.log_textEdit.setGeometry(QtCore.QRect(10, 10, 571, 511))
self.log_textEdit.setReadOnly(True)
self.log_textEdit.setAcceptRichText(False)
self.log_textEdit.setObjectName("log_textEdit")
self.tabWidget.addTab(self.tab_5, "")
MainWindow.setCentralWidget(self.centralwidget)
@ -463,10 +465,10 @@ class Ui_MainWindow(object):
"<p style=\" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;\">导入完毕!</p></body></html>"))
self.import_status_label.setText(_translate("MainWindow", "import_status_label"))
self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab_2), _translate("MainWindow", "执行导入"))
self.time_no_week_checkBox.setText(_translate("MainWindow", "周末不执行"))
self.collect_no_week_checkBox.setText(_translate("MainWindow", "周末不执行"))
self.label_23.setText(_translate("MainWindow", "采集间隔(秒):"))
self.time_start_pushButton.setText(_translate("MainWindow", "启动定时采集"))
self.time_status.setText(_translate("MainWindow", "TextLabel"))
self.collect_start_pushButton.setText(_translate("MainWindow", "启动定时采集"))
self.collect_status_Label.setText(_translate("MainWindow", "TextLabel"))
self.label_21.setText(_translate("MainWindow", "执行时间段1"))
self.label_22.setText(_translate("MainWindow", "-"))
self.label_24.setText(_translate("MainWindow", "执行时间段2"))

View File

@ -735,7 +735,7 @@ p, li { white-space: pre-wrap; }
<attribute name="title">
<string>MySQL定时导入</string>
</attribute>
<widget class="QCheckBox" name="time_no_week_checkBox">
<widget class="QCheckBox" name="collect_no_week_checkBox">
<property name="geometry">
<rect>
<x>40</x>
@ -766,7 +766,7 @@ p, li { white-space: pre-wrap; }
</widget>
</item>
<item>
<widget class="QSpinBox" name="time_sample_spinBox">
<widget class="QSpinBox" name="collect_sample_spinBox">
<property name="maximum">
<number>86400</number>
</property>
@ -774,7 +774,7 @@ p, li { white-space: pre-wrap; }
</item>
</layout>
</widget>
<widget class="QWidget" name="">
<widget class="QWidget" name="layoutWidget">
<property name="geometry">
<rect>
<x>40</x>
@ -785,14 +785,14 @@ p, li { white-space: pre-wrap; }
</property>
<layout class="QHBoxLayout" name="horizontalLayout_11">
<item>
<widget class="QPushButton" name="time_start_pushButton">
<widget class="QPushButton" name="collect_start_pushButton">
<property name="text">
<string>启动定时采集</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="time_status">
<widget class="QLabel" name="collect_status_Label">
<property name="text">
<string>TextLabel</string>
</property>
@ -800,7 +800,7 @@ p, li { white-space: pre-wrap; }
</item>
</layout>
</widget>
<widget class="QWidget" name="">
<widget class="QWidget" name="layoutWidget">
<property name="geometry">
<rect>
<x>40</x>
@ -822,7 +822,7 @@ p, li { white-space: pre-wrap; }
<item>
<layout class="QHBoxLayout" name="horizontalLayout_6">
<item>
<widget class="QTimeEdit" name="time_phase1_start_timeEdit"/>
<widget class="QTimeEdit" name="collect_phase1_start_timeEdit"/>
</item>
<item>
<widget class="QLabel" name="label_22">
@ -832,7 +832,7 @@ p, li { white-space: pre-wrap; }
</widget>
</item>
<item>
<widget class="QTimeEdit" name="time_phase1_last_timeEdit"/>
<widget class="QTimeEdit" name="collect_phase1_last_timeEdit"/>
</item>
</layout>
</item>
@ -850,7 +850,7 @@ p, li { white-space: pre-wrap; }
<item>
<layout class="QHBoxLayout" name="horizontalLayout_7">
<item>
<widget class="QTimeEdit" name="time_phase2_start_timeEdit"/>
<widget class="QTimeEdit" name="collect_phase2_start_timeEdit"/>
</item>
<item>
<widget class="QLabel" name="label_25">
@ -860,7 +860,7 @@ p, li { white-space: pre-wrap; }
</widget>
</item>
<item>
<widget class="QTimeEdit" name="time_phase2_last_timeEdit"/>
<widget class="QTimeEdit" name="collect_phase2_last_timeEdit"/>
</item>
</layout>
</item>
@ -882,6 +882,12 @@ p, li { white-space: pre-wrap; }
<height>511</height>
</rect>
</property>
<property name="readOnly">
<bool>true</bool>
</property>
<property name="acceptRichText">
<bool>false</bool>
</property>
</widget>
</widget>
</widget>

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
import datetime
import mysql.connector
@ -46,6 +47,7 @@ class UsePytdxImportToH5Thread(QThread):
def __init__(self, config):
super(self.__class__, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.msg_name = 'HDF5_IMPORT'
@ -315,4 +317,4 @@ class UsePytdxImportToH5Thread(QThread):
current_progress = (time_progress['SH'] + time_progress['SZ']) // 2
self.send_message([taskname, ktype, current_progress])
else:
print("Unknow task: ", taskname)
self.logger.error("Unknow task: {}".format(taskname))

View File

@ -22,6 +22,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import logging
import sqlite3
from multiprocessing import Queue, Process
from PyQt5.QtCore import QThread, pyqtSignal
@ -37,6 +38,7 @@ class UseTdxImportToH5Thread(QThread):
def __init__(self, config):
super(self.__class__, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.msg_name = 'HDF5_IMPORT'
@ -64,15 +66,38 @@ class UseTdxImportToH5Thread(QThread):
if self.config.getboolean('weight', 'enable', fallback=False):
self.tasks.append(ImportWeightToSqliteTask(self.queue, sqlite_file_name, dest_dir))
if self.config.getboolean('ktype', 'day', fallback=False):
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SH', 'DAY', self.quotations, src_dir, dest_dir))
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SZ', 'DAY', self.quotations, src_dir, dest_dir))
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SH', 'DAY', self.quotations, src_dir, dest_dir
)
)
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SZ', 'DAY', self.quotations, src_dir, dest_dir
)
)
if self.config.getboolean('ktype', 'min5', fallback=False):
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SH', '5MIN', self.quotations, src_dir, dest_dir))
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SZ', '5MIN', self.quotations, src_dir, dest_dir))
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SH', '5MIN', self.quotations, src_dir, dest_dir
)
)
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SZ', '5MIN', self.quotations, src_dir, dest_dir
)
)
if self.config.getboolean('ktype', 'min', fallback=False):
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SH', '1MIN', self.quotations, src_dir, dest_dir))
self.tasks.append(ImportTdxToH5Task(self.queue, sqlite_file_name, 'SZ', '1MIN', self.quotations, src_dir, dest_dir))
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SH', '1MIN', self.quotations, src_dir, dest_dir
)
)
self.tasks.append(
ImportTdxToH5Task(
self.queue, sqlite_file_name, 'SZ', '1MIN', self.quotations, src_dir, dest_dir
)
)
def __del__(self):
for p in self.process_list:
@ -93,8 +118,18 @@ class UseTdxImportToH5Thread(QThread):
def _run(self):
src_dir = self.config['tdx']['dir']
dest_dir = self.config['hdf5']['dir']
hdf5_import_progress = {'SH': {'DAY': 0, '1MIN': 0, '5MIN': 0},
'SZ': {'DAY': 0, '1MIN': 0, '5MIN': 0}}
hdf5_import_progress = {
'SH': {
'DAY': 0,
'1MIN': 0,
'5MIN': 0
},
'SZ': {
'DAY': 0,
'1MIN': 0,
'5MIN': 0
}
}
#正在导入代码表
self.send_message(['START_IMPORT_CODE'])
@ -102,8 +137,12 @@ class UseTdxImportToH5Thread(QThread):
connect = sqlite3.connect(dest_dir + "/stock.db")
create_database(connect)
tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\shm.tnf", 'SH', self.quotations)
tdx_import_stock_name_from_file(connect, src_dir + "\\T0002\\hq_cache\\szm.tnf", 'SZ', self.quotations)
tdx_import_stock_name_from_file(
connect, src_dir + "\\T0002\\hq_cache\\shm.tnf", 'SH', self.quotations
)
tdx_import_stock_name_from_file(
connect, src_dir + "\\T0002\\hq_cache\\szm.tnf", 'SZ', self.quotations
)
self.send_message(['FINISHED_IMPORT_CODE'])
@ -131,7 +170,9 @@ class UseTdxImportToH5Thread(QThread):
self.send_message(['IMPORT_WEIGHT', market, total])
elif taskname == 'IMPORT_KDATA':
hdf5_import_progress[market][ktype] = progress
current_progress = (hdf5_import_progress['SH'][ktype] + hdf5_import_progress['SZ'][ktype]) // 2
current_progress = (
hdf5_import_progress['SH'][ktype] + hdf5_import_progress['SZ'][ktype]
) // 2
self.send_message(['IMPORT_KDATA', ktype, current_progress])
else:
print("Unknow task: ", taskname)
self.logger.error("Unknow task: {}".format(taskname))