sqlite 导入历史财务信息

hikyuu/data/pytdx_finance_to_mysql.py

@@ -125,5 +125,5 @@ def history_finance_import_mysql(connect, filename):
     cur.execute(f"delete from `hku_base`.`HistoryFinance` where file_date={file_date}")
     cur.executemany(
         "insert into `hku_base`.`HistoryFinance` (`file_date`, `market_code`, `report_date`, `values`) values (%s, %s, %s, %s)", ret)
-    cur.close()
     connect.commit()
+    cur.close()

hikyuu/data/pytdx_finance_to_sqlite.py

@@ -22,7 +22,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from hikyuu.data.common import MARKETID, STOCKTYPE
+from hikyuu.data.common import MARKETID, STOCKTYPE, historyfinancialreader
 from hikyuu.data.common_sqlite3 import get_marketid, create_database
 
 
@@ -39,7 +39,7 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     records = []
     for stk in all_list:
         x = pytdx_connect.get_finance_info(1 if stk[1] == MARKETID.SH else 0, stk[2])
-        #print(stk[2])
+        # print(stk[2])
         if x is not None and x['code'] == stk[2]:
             cur.execute(
                 "select updated_date from stkfinance where stockid={} and updated_date={}".format(
@@ -49,9 +49,9 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
             a = cur.fetchall()
             a = [x[0] for x in a]
             if a:
-                #print(a)
+                # print(a)
                 continue
-            #else:
+            # else:
             #    print(market, stk[2])
             records.append(
                 (
@@ -115,6 +115,17 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     return len(records)
 
 
+def history_finance_import_sqlite(connect, filename):
+    file_date = filename[-12:-4]
+    ret = historyfinancialreader(filename)
+    cur = connect.cursor()
+    cur.execute(f"delete from `HistoryFinance` where file_date={file_date}")
+    cur.executemany(
+        "insert into `HistoryFinance` (`file_date`, `market_code`, `report_date`, `values`) values (?,?,?,?)", ret)
+    connect.commit()
+    cur.close()
+
+
 if __name__ == '__main__':
     import os
     import time
@@ -123,7 +134,7 @@ if __name__ == '__main__':
     starttime = time.time()
 
     dest_dir = "d:\\stock"
-    tdx_server = '120.76.152.87'  #'119.147.212.81'
+    tdx_server = '120.76.152.87'  # '119.147.212.81'
     tdx_port = 7709
 
     connect = sqlite3.connect(dest_dir + "\\stock.db")
@@ -148,4 +159,4 @@ if __name__ == '__main__':
     endtime = time.time()
     print("\nTotal time:")
     print("%.2fs" % (endtime - starttime))
-    print("%.2fm" % ((endtime - starttime) / 60))
+    print("%.2fm" % ((endtime - starttime) / 60))

hikyuu/data/sqlite_upgrade/0014.sql

@@ -6,6 +6,19 @@ CREATE TABLE
         `name` VARCHAR(200) NOT NULL,
         PRIMARY KEY(`id`)
     );
+
+CREATE TABLE
+    IF NOT EXISTS `HistoryFinance` (
+        `id` INTEGER,
+        `file_date` INTEGER NOT NULL,
+        `market_code` TEXT NOT NULL,
+        `report_date` INTEGER NOT NULL,
+        `values` BLOB NOT NULL,
+        PRIMARY KEY (`id` AUTOINCREMENT)
+    );
+
+CREATE INDEX "ix1_on_history_finance" ON "HistoryFinance" (file_date);
+CREATE INDEX "ix2_on_history_finance" ON "HistoryFinance" (market_code, report_date);
 	
 INSERT INTO `HistoryFinanceField` (`id`, `name`) VALUES (1, "基本每股收益");
 INSERT INTO `HistoryFinanceField` (`id`, `name`) VALUES (2, "扣除非经常性损益每股收益");

hikyuu/gui/data/ImportHistoryFinanceTask.py

@@ -25,8 +25,11 @@
 import os
 import shutil
 import hashlib
+import sqlite3
+import mysql.connector
 from pytdx.hq import TdxHq_API
 from hikyuu.data.pytdx_finance_to_mysql import history_finance_import_mysql
+from hikyuu.data.pytdx_finance_to_sqlite import history_finance_import_sqlite
 from hikyuu.util import *
 
 
@@ -58,22 +61,27 @@ class ImportHistoryFinanceTask:
         return [l2d(i.strip().split(',')) for i in content]
 
     def import_to_db(self, filename):
-        use_mysql = False
-        if self.config is not None and not self.config.getboolean('hdf5', 'enable', fallback=True):
-            use_mysql = True
-        if not use_mysql:
-            return
+        if self.config.getboolean('hdf5', 'enable', fallback=True):
+            sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
+            connect = sqlite3.connect(sqlite_file, timeout=1800)
+            history_finance_import = history_finance_import_sqlite
+        else:
+            db_config = {
+                'user': self.config['mysql']['usr'],
+                'password': self.config['mysql']['pwd'],
+                'host': self.config['mysql']['host'],
+                'port': self.config['mysql']['port']
+            }
+            connect = mysql.connector.connect(**db_config)
+            history_finance_import = history_finance_import_mysql
 
-        db_config = {
-            'user': self.config['mysql']['usr'],
-            'password': self.config['mysql']['pwd'],
-            'host': self.config['mysql']['host'],
-            'port': self.config['mysql']['port']
-        }
-        import mysql.connector
-        connect = mysql.connector.connect(**db_config)
-        history_finance_import_mysql(connect, filename)
-        connect.close()
+        try:
+            history_finance_import(connect, filename)
+        except Exception as e:
+            hku_error(str(e))
+        finally:
+            connect.commit()
+            connect.close()
 
     def download_file(self, item):
         filename = item['filename']