IPython重构,更改刪除全部变量的方式

This commit is contained in:
hzy15610046011 2020-12-17 11:06:23 +08:00
parent d462ff1be6
commit 84f794cb5b
17 changed files with 160 additions and 145 deletions

View File

@ -1,6 +1,7 @@
import time
import os
import sys
sys.path.append(os.path.dirname(__file__))
import datetime
@ -12,6 +13,7 @@ from qtpy.QtGui import QCloseEvent, QTextCursor, QResizeEvent, QFontDatabase, QM
from qtpy.QtWidgets import QApplication, QTextEdit, QMessageBox, QToolBar, QSplashScreen, QStatusBar
import pmgwidgets
pmgwidgets.in_unit_test = lambda: False
from pmgwidgets import PMGToolBar, ActionWithMessage, PMDockObject, create_icon
@ -25,8 +27,10 @@ from pyminer2.ui.pmwidgets import BaseMainWindow
from pyminer2.globals import get_main_window, get_root_dir, get_application, pythonVersion, openURL
from pyminer2 import globals
t0 = time.time()
from pmlocalserver import server
# TODO:实例化server需要消耗
log_folder = os.path.join(get_root_dir(), 'log')
@ -339,7 +343,7 @@ class MainWindow(BaseMainWindow):
from pyminer2.extensions.extensionlib.extension_lib import extension_lib
extension_lib.get_interface('ipython_console').run_command('Clearing_Variables_ =\'Clear All\'',
hint_text='开始清除...', hidden=False)
extension_lib.get_interface('ipython_console').run_command('__clear_all()', hint_text='清除全部变量', hidden=False)
extension_lib.get_interface('ipython_console').run_command('get_ipython().clear_all()', hint_text='清除全部变量', hidden=False)
def add_toolbar(self, name: str, toolbar: QToolBar,
text: str = 'untitled toolbar'):
@ -669,6 +673,8 @@ def main():
splash.finish(window) # 启动画面结束
window.first_form_display()
logger.info('boot time elapsed:%f s' % (t1 - t0))
logger.info('boot finished at time:' + time.strftime('%H:%M:%S', time.localtime()) + '+%f s' % (
time.time() - int(time.time())))
res = app.exec()
logging.debug("Shutting down, result %d", res)
logging.shutdown()

View File

@ -15,14 +15,10 @@ import os
from typing import Tuple, Dict, Callable
from qtconsole import styles
t0 = time.time()
from qtconsole.manager import QtKernelManager
t2 = time.time()
from qtconsole.rich_jupyter_widget import RichJupyterWidget
t1 = time.time()
from qtconsole.styles import default_light_syntax_style, default_light_style_sheet
print('ipython modules import time', t1 - t0,t2-t0)
from qtpy.QtCore import QObject, Signal, QThread, QWaitCondition, QMutex, QPoint
from qtpy.QtGui import QTextCursor
from qtpy.QtWidgets import QMessageBox, QMenu

View File

@ -1,11 +1,3 @@
import logging
import time
from .io import *
from .debug import *
t3 = time.time()
from .ui import *
t1 = time.time()
print(t1 - t3,'xxxxxxxxxx')

View File

@ -1,3 +1,3 @@
from .baseext import BaseExtension,BaseInterface
from .baseext import BaseExtension, BaseInterface
from .extension_lib import extension_lib
from pyminer2.ui import pmwidgets
from pyminer2.ui import pmwidgets

View File

@ -1,3 +1,5 @@
import time
import os
import inspect
from typing import TYPE_CHECKING, Callable, Dict, List, Tuple, Union, Optional
@ -9,11 +11,14 @@ if TYPE_CHECKING:
def wrapper():
from pyminer2.extensions.extensions_manager.manager import extensions_manager
from pyminer2.workspace.datamanager.datamanager import data_manager
from pyminer2.workspace.datamanager.converter import ConvertError
from pyminer2.workspace.datamanager.exceptions import ConflictError, NotFoundError
from pyminer2.workspace.datamanager.metadataset import WouldBlockError
from pyminer2.globals import get_root_dir, get_main_window, get_application
from pyminer2.ui import pmwidgets
from pyminer2.extensions.extensionlib import baseext
@ -538,3 +543,4 @@ def wrapper():
extension_lib = wrapper()

View File

@ -3,6 +3,8 @@
说明
dialog无需写在json里面直接调用主界面的控件就可以了
"""
import time
import os
import sys
from pyminer2.extensions.extensionlib import BaseExtension, BaseInterface
@ -22,3 +24,4 @@ class Extension(BaseExtension):
class Interface(BaseInterface):
pass

View File

@ -9,7 +9,6 @@ from typing import Dict, Union
from qtpy.QtCore import QLocale
from qtpy.QtWidgets import QApplication
sys.path.append(os.path.dirname(__file__))
from pyminer2.extensions.extensionlib import BaseInterface, BaseExtension
@ -17,12 +16,10 @@ from .codeeditor.tabwidget import PMCodeEditTabWidget
from .debugger import PMDebugConsoleTabWidget
from .toolbar import PMEditorToolbar
from pmgwidgets import PMGPanel, create_file_if_not_exist, load_json, dump_json
import json
class Extension(BaseExtension):
def __init__(self):
super(Extension, self).__init__()
@ -122,16 +119,16 @@ class Extension(BaseExtension):
settings = self.settings
new_settings = [
('line_ctrl', 'encoding_declaration_text', 'Encoding Declaration', settings['encoding_declaration_text']),
('numberspin_ctrl', 'font_size', 'Font Size', settings['font_size'], '', (5, 25), 1),
('check_ctrl', 'check_syntax_background', 'Check Syntax Background',
('line_ctrl', 'encoding_declaration_text', '编码声明', settings['encoding_declaration_text']),
('numberspin_ctrl', 'font_size', '字体大小', settings['font_size'], '', (5, 25), 1),
('check_ctrl', 'check_syntax_background', '后台语法检查', # Check Syntax Background',
settings['check_syntax_background']),
('check_ctrl', 'smart_autocomp_on', 'Smart Autocompletion', settings['smart_autocomp_on']),
('check_ctrl', 'wrap', 'Wrap', settings['wrap']),
('keymap_ctrl', 'key_comment', 'Comment/Uncomment', settings['key_comment']),
('check_ctrl', 'smart_autocomp_on', '智能自动补全jedi', settings['smart_autocomp_on']),
('check_ctrl', 'wrap', '自动换行', settings['wrap']),
('keymap_ctrl', 'key_comment', '注释/取消注释快捷键', settings['key_comment']),
]
self.update_settings(settings)
panel: 'PMGPanel' = self.extension_lib.Program.add_settings_panel('Editor', new_settings)
panel: 'PMGPanel' = self.extension_lib.Program.add_settings_panel('编辑器', new_settings)
panel.signal_settings_changed.connect(self.update_settings)
panel.get_ctrl('font_size').setEnabled(False)

View File

@ -1,6 +1,7 @@
"""
此文件在IPython启动之时输入到IPython的变量空间中
它的作用是预先定义一些函数和魔术方法定义与工作空间通信的方法
之后重构可以考虑直接重写IPython.core.interactiveshell.InteractiveShell的各个方法这样比在这里装配要好一些
"""
import numpy as np
@ -8,17 +9,19 @@ import matplotlib.pyplot as plt
import pandas as pd
import ast
from IPython.core.magic import register_line_magic, register_cell_magic, register_line_cell_magic
import types
# from matgen import v_, h_, M_
from pyminer2.core import *
from pmgwidgets import BaseClient
import typing
if typing.TYPE_CHECKING:
from IPython.core.interactiveshell import InteractiveShell
from IPython.core.getipython import get_ipython
__builtins = [__k for __k in globals().keys()]
__ip = get_ipython()
__ip: 'InteractiveShell' = get_ipython()
__ip.builtin_vars = [__k for __k in globals().keys()]
__ip.builtin_values = {__k: __v for __k, __v in globals().items() if not __k.startswith('__')}
__ip.var_name_list = []
__var_name_list = []
__neglect_post_run = False # 为True的时候执行命令不触发回调。防止刷新造成无限反复。
__ip.neglect_post_run = False
@ -44,18 +47,19 @@ def __refresh_vars():
刷新工作空间的变量并且删除在ipython不存在的变量
:return:
"""
global __var_name_list
from pmgwidgets import BaseClient
import types
__ip = get_ipython()
__client = BaseClient()
__data_message = {}
__data = {
__k: __v for __k, __v in globals().items() if
__k not in __builtins and not __k.startswith('_') and not isinstance(
__v,
types.ModuleType)}
__data = __filter_vars(__data)
__k not in __ip.builtin_vars and not __k.startswith('_') and not isinstance(__v, types.ModuleType)
}
__data = __ip.filter_vars(__data)
__name_list = list(__data.keys())
__deleted_data_name = set(__var_name_list) - set(__name_list)
__var_name_list = __name_list
__deleted_data_name = set(__ip.var_name_list) - set(__name_list)
__ip.var_name_list = __name_list
__existing_vars = __client.get_all_public_var_names()
for __deleted_name in __deleted_data_name:
@ -64,6 +68,7 @@ def __refresh_vars():
def __update_globals_from_workspace(variables: list = None):
from pmgwidgets import BaseClient
__client = BaseClient()
if variables is None:
globals().update(__client.get_all_vars())
@ -77,13 +82,14 @@ def __delete_var(__var_name: str):
:param __var_name:
:return:
"""
global __neglect_post_run, __var_name_list
import types
__ip = get_ipython()
if __var_name in globals().keys():
__unused = globals().pop(__var_name)
__neglect_post_run = True
__ip.neglect_post_run = True
__data = {__k: __v for __k, __v in globals().items()
if __k not in __builtins and not __k.startswith('_') and not isinstance(__v, types.ModuleType)}
__var_name_list = list(__data.keys())
if __k not in __ip.builtin_vars and not __k.startswith('_') and not isinstance(__v, types.ModuleType)}
__ip.var_name_list = list(__data.keys())
def __post_run_callback():
@ -91,16 +97,18 @@ def __post_run_callback():
IPython代码执行后触发的回调函数
:return:
"""
global __neglect_post_run
if not __neglect_post_run:
__refresh_vars()
__ip = get_ipython()
if not __ip.neglect_post_run:
__ip.refresh_vars()
else:
__neglect_post_run = False
__ip.neglect_post_run = False
def __is_transfer_allowed(__key: str) -> bool:
return __key not in __builtins and not __key.startswith('_') and not isinstance(globals().get(__key),
types.ModuleType)
import types
__ip = get_ipython()
return __key not in __ip.builtin_vars and not __key.startswith('_') and not isinstance(globals().get(__key),
types.ModuleType)
def __pre_run_callback():
@ -108,15 +116,15 @@ def __pre_run_callback():
IPython代码执行前触发的回调函数
:return:
"""
global __neglect_post_run, __var_name_list
if not __neglect_post_run:
import types
__ip = get_ipython()
if not __ip.neglect_post_run:
__data = {__k: __v for __k, __v in globals().items()
if __k not in __builtins and not __k.startswith('_') and not isinstance(__v, types.ModuleType)}
__var_name_list = list(__data.keys())
if __k not in __ip.builtin_vars and not __k.startswith('_') and not isinstance(__v, types.ModuleType)}
__ip.var_name_list = list(__data.keys())
# __update_globals_from_workspace()
else:
__neglect_post_run = True
__ip.neglect_post_run = True
@register_line_cell_magic
@ -141,9 +149,9 @@ def __clear_all():
Returns:
"""
global __var_name_list, __neglect_post_run
for __var_name in __var_name_list:
from pmgwidgets import BaseClient
__ip = get_ipython()
for __var_name in __ip.var_name_list:
globals().pop(__var_name)
__client = BaseClient()
vars = __client.get_all_public_var_names()
@ -152,10 +160,10 @@ def __clear_all():
import time as _time
_time.sleep(0.1)
__neglect_post_run = True
__ip.neglect_post_run = True
__run_cell_func = __ip.run_cell
__ip.original_run_cell_func = __ip.run_cell
def __cell_exec_func(raw_cell, store_history=False, silent=False, shell_futures=True):
@ -170,12 +178,58 @@ def __cell_exec_func(raw_cell, store_history=False, silent=False, shell_futures=
Returns:
"""
import ast
from pmgwidgets import BaseClient
__ip = get_ipython()
class PyMinerIPyConsoleNodeTransformer(ast.NodeTransformer):
def __init__(self):
super(PyMinerIPyConsoleNodeTransformer, self).__init__()
self.identifier_list = []
self.str_list = []
def visit_Name(self, node: ast.Name):
"""
Args:
node:
Returns:
"""
self.identifier_list.append(node.id)
def visit_Str(self, node: ast.Str):
"""
Args:
node:
Returns:
"""
self.str_list.append(node.s)
def show_identifiers_might_changed(self, code):
"""
Args:
code:
Returns:
"""
self.identifier_list = []
self.str_list = []
self.visit(ast.parse(code))
return [s for s in list(set(self.identifier_list + self.str_list)) if s.isidentifier()]
try:
__identifiers = PyMinerIPyConsoleNodeTransformer().show_identifiers_might_changed(raw_cell)
__identifiers_in_ws = set(BaseClient().get_all_public_var_names()) # identifiers in workspace
__identifiers_to_update = [__identifier for __identifier in __identifiers if
(__identifier in __identifiers_in_ws)]
__update_globals_from_workspace(__identifiers_to_update)
__ip.update_globals_from_workspace(__identifiers_to_update)
except SyntaxError:
__identifiers = []
except:
@ -183,59 +237,34 @@ def __cell_exec_func(raw_cell, store_history=False, silent=False, shell_futures=
traceback.print_exc()
# print('出现内部错误请右键菜单重启IPython控制台。')
s = __run_cell_func(raw_cell, store_history=store_history, silent=silent, shell_futures=shell_futures)
s = get_ipython().original_run_cell_func(raw_cell, store_history=store_history, silent=silent,
shell_futures=shell_futures)
__var_dic_keys = [__identifier for __identifier in __identifiers if
__identifier in globals().keys() and __is_transfer_allowed(__identifier)]
__identifier in globals().keys() and __ip.is_transfer_allowed(__identifier)]
__var_dic = {__k: globals()[__k] for __k in __var_dic_keys}
BaseClient().set_var_dic(__var_dic, 'ipython')
return s
class PyMinerIPyConsoleNodeTransformer(ast.NodeTransformer):
def __init__(self):
super(PyMinerIPyConsoleNodeTransformer, self).__init__()
self.identifier_list = []
self.str_list = []
def __reset(new_session=True):
__ip = get_ipython()
__ip.original_reset_func(new_session)
def visit_Name(self, node: ast.Name):
"""
Args:
node:
Returns:
"""
self.identifier_list.append(node.id)
def visit_Str(self, node: ast.Str):
"""
Args:
node:
Returns:
"""
self.str_list.append(node.s)
def show_identifiers_might_changed(self, code):
"""
Args:
code:
Returns:
"""
self.identifier_list = []
self.str_list = []
self.visit(ast.parse(code))
return [s for s in list(set(self.identifier_list + self.str_list)) if s.isidentifier()]
globals().update(__ip.builtin_values)
__ip.filter_vars = __filter_vars
__ip.refresh_vars = __refresh_vars
__ip.update_globals_from_workspace = __update_globals_from_workspace
__ip.delete_var = __delete_var
__ip.post_run_callback = __post_run_callback
__ip.is_transfer_allowed = __is_transfer_allowed
__ip.pre_run_callback = __pre_run_callback
__ip.clear_all = __clear_all
__ip.run_cell = __cell_exec_func
__ip.original_reset_func = __ip.reset
__ip.reset = __reset # lambda new_session:print('123123123')
__ip.events.register('post_run_cell', __post_run_callback)
# __ip.events.register('pre_execute', __pre_execute)
# __ip.events.register('pre_run_cell', __pre_run_callback)

View File

@ -11,11 +11,10 @@ Created on 2020/8/24
import os
from typing import Tuple
import time
t0 = time.time()
from pmgwidgets import PMDockObject, get_ipython_console_class, in_unit_test
PMGIpythonConsole = get_ipython_console_class()
t1 = time.time()
print(t1-t0)
class ConsoleWidget(PMGIpythonConsole, PMDockObject):
def __init__(self, *args, **kwargs):

View File

@ -6,8 +6,6 @@ import time
from typing import Dict, Any, List
import cloudpickle
import numpy
import pandas
from qtpy.QtCore import Signal, QTimer
from pmgwidgets import PMGServer
@ -62,6 +60,7 @@ class PMServer(PMGServer):
return json.dumps({'message': 'succeeded', 'var_dic': data_b64})
def get_all_var_names_of_type(self, type):
import pandas,numpy
def istable(var):
return isinstance(var, pandas.DataFrame) or isinstance(var, numpy.ndarray) and len(var.shape) == 2

View File

@ -11,11 +11,8 @@ from qtpy.QtCore import Signal, Qt, QLocale
from pmgwidgets import PMDockObject, in_unit_test, create_translator
t0 = time.time()
from pmtoolbox import load_variable_pmd, load_variable_pkl, save_variable_pkl, save_variable_pmd, save_variable_table, \
save_variable_matrix
t1 = time.time()
print(__file__,t1-t0)
from pyminer2.extensions.extensionlib import BaseExtension, BaseInterface
if TYPE_CHECKING:
@ -489,7 +486,6 @@ class PMWorkspaceInspectWidget(QWidget, PMDockObject):
self.extension_lib = extension_lib
self.data = self.extension_lib.Data.get_all_variables()
self.var_tree.set_data(self.extension_lib.Data.get_all_public_variables())
print(threading.currentThread())
def on_changed(varname: str, variable, data_source: str):
self.data[varname] = variable

View File

@ -1,24 +1,19 @@
import os
import threading
import time
from typing import Dict, TYPE_CHECKING, Callable, Any
from qtpy.QtWidgets import QTreeWidget, QTreeWidgetItem, QWidget, QVBoxLayout, QMenu, \
QMessageBox, QFileDialog, QInputDialog, QApplication
from qtpy.QtCore import Signal, Qt, QLocale
from pyminer2.extensions.extensionlib import BaseExtension, BaseInterface
t0 = time.time()
if TYPE_CHECKING:
from .data_viewer import PMVariableViewerWidget
from .inspectortable import PMWorkspaceInspectWidget
else:
from .inspectortable import PMWorkspaceInspectWidget
t1 = time.time()
print('workspace_inspector', t1-t0)
class Extension(BaseExtension):
interace: 'Interface' = None

View File

@ -39,6 +39,9 @@ class OptionForm(QDialog, Option_Ui_Form):
super().__init__(parent)
self.setupUi(self)
self.center()
self.page_format.setEnabled(False)
self.page_appearance.setEnabled(False)
self.page_interpreter.setEnabled(False)
self.interpreter_table: QTableWidget = self.tableWidget
self.setup_ui()
@ -53,18 +56,7 @@ class OptionForm(QDialog, Option_Ui_Form):
self.toolButton_output.clicked.connect(self.slot_change_output)
self.pushButton_cancel.clicked.connect(self.close)
self.pushButton_ok.clicked.connect(self.close)
interpreter_settings = [
('line_ctrl', 'interpreter_name', 'Interpreter Name', 'Unknown')
]
new_settings = [('line_ctrl', 'dataserver_address', 'Data Server Address', 'http://localhost:8888'),
('number_ctrl', 'dataserver_port', 'Data Server Port', 12306, '', (0, 65535)),
('check_ctrl', 'save_layout_on_close', 'Save Layout On Close', True),
('combo_ctrl', 'theme', 'Theme', 'Fusion',
['Fusion', 'Qdarkstyle', 'windows', 'windowsvista'])]
self.add_settings_panel('网络端口设置面板', new_settings)
self.pushButton_help.clicked.connect(self.get_help)
def setup_ui(self):
self.comboBox_9.setEnabled(False)
@ -111,7 +103,7 @@ class OptionForm(QDialog, Option_Ui_Form):
self.refresh_settings()
def slot_change_workspace(self):
directory = QFileDialog.getExistingDirectory(self, "选择工作区间位置", os.path.expanduser('~'))
directory = QFileDialog.getExistingDirectory(self, "选择工作区间位置", )
if not directory == '':
self.label_workspace.setText(directory)
@ -156,6 +148,9 @@ class OptionForm(QDialog, Option_Ui_Form):
self.load_settings()
super(OptionForm, self).exec()
def get_help(self):
webbrowser.open('https://gitee.com/py2cn/pyminer/wikis/%E9%85%8D%E7%BD%AEPyMiner?sort_id=3263840')
class AppstoreForm(QWidget, appStore_Ui_Form):
def __init__(self):
@ -206,6 +201,7 @@ class MarketPlaceInstall(QWidget, marketplace_install_Ui_Form):
qr.moveCenter(cp)
self.move(qr.topLeft())
class MarketPlace(QWidget, marketplace_Ui_Form):
def __init__(self):
super(MarketPlace, self).__init__()
@ -257,11 +253,10 @@ class MarketPlace(QWidget, marketplace_Ui_Form):
self.tableWidget.setItem(i, j, item)
def pip_install_display(self):
pm_pack_install=MarketPlaceInstall()
pm_pack_install = MarketPlaceInstall()
pm_pack_install.show()
class AboutForm(QWidget, About_Ui_Form):
def __init__(self):
super(AboutForm, self).__init__()

View File

@ -1,5 +1,7 @@
import numpy as np
import pandas as pd
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import numpy as np
import pandas as pd
from pyminer2.workspace.datamanager.variable import Variable
from .exceptions import ConvertError
@ -32,8 +34,9 @@ class Converter:
# convert to data, func format: convert_obj
def convert_ndarray(self, arr: np.ndarray) -> dict:
def convert_ndarray(self, arr: 'np.ndarray') -> dict:
# TODO (panhaoyu) 三维数组甚至四维数组都是很常见的数据格式,应该支持
import numpy as np
if arr.dtype in (np.int, np.float):
if len(arr.shape) == 2:
return Variable('Matrix', {'value': arr.tolist()}).dump()
@ -45,22 +48,26 @@ class Converter:
raise ConvertError(f'{arr} is inconvertible')
def convert_list(self, lst: list) -> dict:
import numpy as np
return self.convert_ndarray(np.array(lst))
def convert_dataframe(self, dataframe: pd.DataFrame) -> dict:
def convert_dataframe(self, dataframe: 'pd.DataFrame') -> dict:
return Variable('DataFrame', {'table': dataframe.values.tolist(), 'columns': dataframe.columns.tolist()})
# convert to var, func format: iconvert_type
# TODO (panhaoyu) 这三个函数目前没有在pycharm中发现调用是否可以删除
def iconvert_matrix(self, mat: Variable) -> np.ndarray:
def iconvert_matrix(self, mat: Variable) ->'np.ndarray':
import numpy as np
assert mat.type == 'Matrix'
return np.array(mat['value'])
def iconvert_vector(self, vec: Variable) -> np.ndarray:
def iconvert_vector(self, vec: Variable) -> 'np.ndarray':
import numpy as np
assert vec.type == 'Vector'
return np.array(vec['value'])
def iconvert_dataframe(self, dataframe: Variable) -> pd.DataFrame:
def iconvert_dataframe(self, dataframe: Variable) -> 'pd.DataFrame':
import pandas as pd
assert dataframe.type == 'DataFrame'
return pd.DataFrame(dataframe['table'], columns=dataframe['columns'])

View File

@ -1,5 +1,4 @@
from typing import Dict, Union
from pyminer2.workspace.datamanager.converter import Converter
from pyminer2.workspace.datamanager.dataset import DataSet
from pyminer2.workspace.datamanager.exceptions import NotFoundError, ConflictError

View File

@ -553,7 +553,6 @@ class IPyLexer(Lexer):
for token in lex.get_tokens_unprocessed(text):
yield token
####################
print('frontend widget time', time.time() - t0)
from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound

View File

@ -11,7 +11,6 @@ from qtpy import QtCore, QtGui, QtWidgets
from ipython_genutils.path import ensure_dir_exists
from traitlets import Bool
from qtconsole.svg import save_svg, svg_to_clipboard, svg_to_image
t0 = time.time()
from .jupyter_widget import JupyterWidget
@ -22,8 +21,6 @@ try:
except ImportError:
latex_to_png = None
t1 = time.time()
print('rich jupyter widget time',t1-t0)
class LatexError(Exception):
"""Exception for Latex errors"""