mirror of
https://gitee.com/fastnlp/fastNLP.git
synced 2024-12-02 04:07:35 +08:00
Merge branch 'dev0.8.0' of github.com:fastnlp/fastNLP into dev0.8.0
This commit is contained in:
yh_cc
commit
8c8cf70959
@ -6,7 +6,7 @@ SPHINXOPTS =
|
||||
SPHINXAPIDOC = sphinx-apidoc
|
||||
SPHINXBUILD = sphinx-build
|
||||
SPHINXPROJ = fastNLP
|
||||
SPHINXEXCLUDE = ../fastNLP/transformers/* ../fastNLP/modules/* ../fastNLP/core/drivers/torch_paddle_driver/* ../fastNLP/core/utils/torch_paddle_utils.py
|
||||
SPHINXEXCLUDE = ../fastNLP/transformers/*
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
PORT = 9000
|
||||
@ -16,7 +16,7 @@ help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
|
||||
|
||||
apidoc:
|
||||
$(SPHINXAPIDOC) -efM -d 6 -o source ../$(SPHINXPROJ) $(SPHINXEXCLUDE)
|
||||
$(SPHINXAPIDOC) -efM -o source ../$(SPHINXPROJ) $(SPHINXEXCLUDE)
|
||||
|
||||
server:
|
||||
cd build/html && python -m http.server $(PORT)
|
||||
@ -24,6 +24,9 @@ server:
|
||||
delete:
|
||||
rm -f source/$(SPHINXPROJ).* source/modules.rst && rm -rf build
|
||||
|
||||
web:
|
||||
make html && make server
|
||||
|
||||
dev:
|
||||
make delete && make apidoc && make html && make server
|
||||
|
||||
|
||||
@ -42,7 +42,8 @@ extensions = [
|
||||
'sphinx.ext.viewcode',
|
||||
'sphinx.ext.autosummary',
|
||||
'sphinx.ext.mathjax',
|
||||
'sphinx.ext.todo'
|
||||
'sphinx.ext.todo',
|
||||
'sphinx_autodoc_typehints'
|
||||
]
|
||||
|
||||
autodoc_default_options = {
|
||||
@ -53,8 +54,10 @@ autodoc_default_options = {
|
||||
|
||||
add_module_names = False
|
||||
autosummary_ignore_module_all = False
|
||||
autodoc_typehints = "description"
|
||||
# autodoc_typehints = "description"
|
||||
autoclass_content = "class"
|
||||
typehints_fully_qualified = False
|
||||
typehints_defaults = "comma"
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
@ -168,8 +171,8 @@ texinfo_documents = [
|
||||
|
||||
# -- Extension configuration -------------------------------------------------
|
||||
def maybe_skip_member(app, what, name, obj, skip, options):
|
||||
# if obj.__doc__ is None:
|
||||
# return True
|
||||
if obj.__doc__ is None:
|
||||
return True
|
||||
if name == "__init__":
|
||||
return False
|
||||
if name.startswith("_"):
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.callbacks.torch_callbacks
|
||||
|
||||
@ -18,7 +18,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.callbacks.callback
|
||||
fastNLP.core.callbacks.callback_event
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.callbacks.torch_callbacks.torch_grad_clip_callback
|
||||
fastNLP.core.callbacks.torch_callbacks.torch_lr_sched_callback
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.collators.padders.exceptions
|
||||
fastNLP.core.collators.padders.get_padder
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.collators.padders
|
||||
|
||||
@ -18,7 +18,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.collators.collator
|
||||
fastNLP.core.collators.packer_unpacker
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.controllers.loops.evaluate_batch_loop
|
||||
fastNLP.core.controllers.loops.loop
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.controllers.loops
|
||||
fastNLP.core.controllers.utils
|
||||
@ -19,7 +19,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.controllers.evaluator
|
||||
fastNLP.core.controllers.trainer
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.controllers.utils.state
|
||||
fastNLP.core.controllers.utils.utils
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataloaders.jittor_dataloader.fdl
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataloaders.paddle_dataloader.fdl
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
fastNLP.core.dataloaders.prepare\_dataloader module
|
||||
===================================================
|
||||
|
||||
.. automodule:: fastNLP.core.dataloaders.prepare_dataloader
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataloaders.jittor_dataloader
|
||||
fastNLP.core.dataloaders.paddle_dataloader
|
||||
@ -20,7 +20,8 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataloaders.mix_dataloader
|
||||
fastNLP.core.dataloaders.prepare_dataloader
|
||||
fastNLP.core.dataloaders.utils
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataloaders.torch_dataloader.fdl
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.dataset.dataset
|
||||
fastNLP.core.dataset.field
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.drivers.jittor_driver.initialize_jittor_driver
|
||||
fastNLP.core.drivers.jittor_driver.jittor_driver
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.drivers.paddle_driver.dist_utils
|
||||
fastNLP.core.drivers.paddle_driver.fleet
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.drivers.jittor_driver
|
||||
fastNLP.core.drivers.paddle_driver
|
||||
@ -20,7 +20,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.drivers.choose_driver
|
||||
fastNLP.core.drivers.driver
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.drivers.torch_driver.ddp
|
||||
fastNLP.core.drivers.torch_driver.dist_utils
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.log.handler
|
||||
fastNLP.core.log.highlighter
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend.jittor_backend.backend
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend.paddle_backend.backend
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend.jittor_backend
|
||||
fastNLP.core.metrics.backend.paddle_backend
|
||||
@ -20,7 +20,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend.auto_backend
|
||||
fastNLP.core.metrics.backend.backend
|
||||
|
||||
@ -10,6 +10,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend.torch_backend.backend
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.backend
|
||||
|
||||
@ -18,7 +18,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.metrics.accuracy
|
||||
fastNLP.core.metrics.classify_f1_pre_rec_metric
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.callbacks
|
||||
fastNLP.core.collators
|
||||
@ -27,6 +27,6 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.vocabulary
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.samplers.conversion_utils
|
||||
fastNLP.core.samplers.mix_sampler
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core.utils.cache_results
|
||||
fastNLP.core.utils.dummy_class
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.envs.distributed
|
||||
fastNLP.envs.env
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.io.loader.classification
|
||||
fastNLP.io.loader.conll
|
||||
|
||||
@ -10,7 +10,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.io.pipe.classification
|
||||
fastNLP.io.pipe.conll
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.io.loader
|
||||
fastNLP.io.pipe
|
||||
@ -19,7 +19,7 @@ Submodules
|
||||
----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.io.data_bundle
|
||||
fastNLP.io.embed_loader
|
||||
|
||||
@ -10,7 +10,7 @@ Subpackages
|
||||
-----------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP.core
|
||||
fastNLP.envs
|
||||
|
||||
@ -2,6 +2,6 @@ fastNLP
|
||||
=======
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 6
|
||||
:maxdepth: 4
|
||||
|
||||
fastNLP
|
||||
|
||||
@ -63,7 +63,6 @@ __all__ = [
|
||||
"PaddleFleetDriver",
|
||||
"JittorSingleDriver",
|
||||
"JittorMPIDriver",
|
||||
"TorchPaddleDriver",
|
||||
|
||||
# log
|
||||
"logger",
|
||||
|
||||
@ -8,8 +8,8 @@ from fastNLP.core.utils.utils import _get_fun_msg
|
||||
|
||||
def _get_monitor_value(monitor: Union[callable, str], real_monitor: Optional[str], res: dict) ->Tuple[str, float]:
|
||||
"""
|
||||
从res中寻找 monitor 并返回。如果 monitor 没找到则尝试用 _real_monitor ,若 _real_monitor 为 None 则尝试使用 monitor 的值进行
|
||||
匹配。
|
||||
从 ``res`` 中寻找 ``monitor`` 并返回。如果 ``monitor`` 没找到则尝试用 ``_real_monitor`` ,若 ``_real_monitor`` 为 ``None``
|
||||
则尝试使用 ``monitor`` 的值进行匹配。
|
||||
|
||||
:param monitor:
|
||||
:param real_monitor:
|
||||
|
||||
@ -162,9 +162,9 @@ class PaddleDataLoader(DataLoader):
|
||||
|
||||
def get_batch_indices(self) -> List[int]:
|
||||
"""
|
||||
获取当前 batch 的 idx
|
||||
获取当前 ``batch`` 中每条数据对应的索引。
|
||||
|
||||
:return:
|
||||
:return: 当前 ``batch`` 数据的索引
|
||||
"""
|
||||
return self.cur_batch_indices
|
||||
|
||||
|
||||
@ -170,9 +170,9 @@ class TorchDataLoader(DataLoader):
|
||||
|
||||
def get_batch_indices(self) -> List[int]:
|
||||
"""
|
||||
获取当前 batch 的 idx
|
||||
获取当前 ``batch`` 中每条数据对应的索引。
|
||||
|
||||
:return:
|
||||
:return: 当前 ``batch`` 数据的索引
|
||||
"""
|
||||
return self.cur_batch_indices
|
||||
|
||||
|
||||
@ -400,15 +400,16 @@ class DataSet:
|
||||
new_field_name: str = None, num_proc: int = 0,
|
||||
progress_desc: str = None, show_progress_bar: bool = True):
|
||||
r"""
|
||||
将 DataSet 中的每个 instance 中的名为 `field_name` 的 field 传给 func，并获取它的返回值。
|
||||
将 :class:`~DataSet` 每个 ``instance`` 中为 ``field_name`` 的 ``field`` 传给函数 ``func``，并获取函数的返回值。
|
||||
|
||||
:param field_name: 传入 func 的是哪个 field。
|
||||
:param func: input是 instance 中名为 `field_name` 的 field 的内容。
|
||||
:param new_field_name: 将 func 返回的内容放入到 `new_field_name` 这个 field 中，如果名称与已有的 field 相同，则覆
|
||||
盖之前的 field。如果为 None 则不创建新的 field。
|
||||
:param num_proc: 进程的数量。请注意，由于python语言的特性，多少进程就会导致多少倍内存的增长。
|
||||
:param progress_desc: progress_desc 的值，默认为 Main
|
||||
:param show_progress_bar: 是否展示进度条，默认展示进度条
|
||||
:param field_name: 传入 ``func`` 的 ``field`` 名称。
|
||||
:param func: 一个函数，其输入是 ``instance`` 中名为 ``field_name`` 的 ``field`` 的内容。
|
||||
:param new_field_name: 将 ``func`` 返回的内容放入到 ``new_field_name`` 对应的 ``field`` 中，如果名称与已有的 ``field`` 相同
|
||||
则进行覆盖。如果为 ``None`` 则不会覆盖和创建 ``field`` 。
|
||||
:param num_proc: 使用进程的数量。请注意，由于 ``python`` 语言的特性，使用了多少进程就会导致多少倍内存的增长。
|
||||
:param progress_desc: 进度条的描述字符，默认为 ``Main``。
|
||||
:param show_progress_bar: 是否展示进度条；默认为展示。
|
||||
:return: 从函数 ``func`` 中得到的返回值。
|
||||
"""
|
||||
assert len(self) != 0, "Null DataSet cannot use apply_field()."
|
||||
if not self.has_field(field_name=field_name):
|
||||
|
||||
@ -9,7 +9,6 @@ __all__ = [
|
||||
"JittorDriver",
|
||||
"JittorSingleDriver",
|
||||
"JittorMPIDriver",
|
||||
"TorchPaddleDriver",
|
||||
'torch_seed_everything',
|
||||
'paddle_seed_everything',
|
||||
'optimizer_state_to_device'
|
||||
@ -18,7 +17,6 @@ __all__ = [
|
||||
from .torch_driver import TorchDriver, TorchSingleDriver, TorchDDPDriver, torch_seed_everything, optimizer_state_to_device
|
||||
from .jittor_driver import JittorDriver, JittorMPIDriver, JittorSingleDriver
|
||||
from .paddle_driver import PaddleDriver, PaddleFleetDriver, PaddleSingleDriver, paddle_seed_everything
|
||||
from .torch_paddle_driver import TorchPaddleDriver
|
||||
from .driver import Driver
|
||||
|
||||
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
__all__ = [
|
||||
"TorchPaddleDriver",
|
||||
]
|
||||
|
||||
from .torch_paddle_driver import TorchPaddleDriver
|
||||
@ -1,193 +0,0 @@
|
||||
from typing import Optional, Dict, Union, Callable, Tuple
|
||||
|
||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
|
||||
from fastNLP.core.utils.utils import _get_fun_msg
|
||||
|
||||
|
||||
if _NEED_IMPORT_PADDLE:
|
||||
import paddle
|
||||
from paddle.io import DataLoader as PaddleDataLoader
|
||||
from paddle.optimizer import Optimizer as PaddleOptimizer
|
||||
|
||||
if _NEED_IMPORT_TORCH:
|
||||
import torch
|
||||
from torch.utils.data import DataLoader as TorchDataLoader
|
||||
from torch.optim import Optimizer as TorchOptimizer
|
||||
|
||||
from fastNLP.core.drivers.driver import Driver
|
||||
from fastNLP.envs.distributed import rank_zero_call
|
||||
from fastNLP.core.utils.utils import auto_param_call, apply_to_collection
|
||||
from fastNLP.core.log.logger import logger
|
||||
from fastNLP.modules.mix_modules.mix_module import MixModule
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TorchPaddleDriver",
|
||||
]
|
||||
|
||||
class TorchPaddleDriver(Driver):
|
||||
"""
|
||||
针对torch和paddle混合模型的driver
|
||||
由于是两种不同的框架不方便实现多卡，暂时先实现CPU和GPU单卡的功能
|
||||
"""
|
||||
def __init__(self, model, device: Optional[str] = None, **kwargs):
|
||||
super(TorchPaddleDriver, self).__init__(model)
|
||||
|
||||
self.model_device = device
|
||||
self.torch_non_blocking = kwargs.get("torch_non_blocking", None)
|
||||
self.paddle_blocking = kwargs.get("paddle_blocking", None)
|
||||
|
||||
self._data_device = kwargs.get("_data_device", None)
|
||||
if isinstance(self._data_device, int):
|
||||
# 将data_device设置为cuda:x的字符串形式
|
||||
if self._data_device < 0:
|
||||
raise ValueError("Parameter `_data_device` can not be smaller than 0.")
|
||||
_could_use_device_num = paddle.device.cuda.device_count()
|
||||
if self._data_device >= _could_use_device_num:
|
||||
raise ValueError("The gpu device that parameter `device` specifies is not existed.")
|
||||
self._data_device = f"cuda:{self._data_device}"
|
||||
elif self._data_device is not None:
|
||||
raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.")
|
||||
|
||||
def setup(self):
|
||||
if self.model_device is not None:
|
||||
paddle.device.set_device(self.model_device.replace("cuda", "gpu"))
|
||||
self.model.to(self.model_device)
|
||||
|
||||
@staticmethod
|
||||
def check_dataloader_legality(dataloader, dataloader_name, is_train: bool = False):
|
||||
if is_train:
|
||||
if not isinstance(dataloader, (TorchDataLoader, PaddleDataLoader)):
|
||||
raise ValueError(f"Parameter `{dataloader_name}` should be 'torch.util.data.DataLoader' or `paddle.io.dataloader` type, not {type(dataloader)}.")
|
||||
else:
|
||||
if not isinstance(dataloader, Dict):
|
||||
raise ValueError(f"Parameter `{dataloader_name}` should be 'Dict' type, not {type(dataloader)}.")
|
||||
else:
|
||||
for each_dataloader in dataloader.values():
|
||||
if not isinstance(each_dataloader, (TorchDataLoader, PaddleDataLoader)):
|
||||
raise ValueError(f"Each dataloader of parameter `{dataloader_name}` should be "
|
||||
f"'torch.util.data.DataLoader' or `paddle.io.dataloader` "
|
||||
f"type, not {type(each_dataloader)}.")
|
||||
|
||||
@staticmethod
|
||||
def _check_optimizer_legality(optimizers):
|
||||
for each_optimizer in optimizers:
|
||||
if not isinstance(each_optimizer, (TorchOptimizer, PaddleOptimizer)):
|
||||
raise ValueError(f"Each optimizers of parameter `optimizers` should be "
|
||||
f"'torch.optim.Optimizer' or 'paddle.optimizers.Optimizer' type, "
|
||||
f"not {type(each_optimizer)}.")
|
||||
|
||||
def step(self):
|
||||
for optimizer in self.optimizers:
|
||||
optimizer.step()
|
||||
|
||||
def backward(self, loss):
|
||||
loss.backward()
|
||||
|
||||
def zero_grad(self):
|
||||
for optimizer in self.optimizers:
|
||||
if isinstance(optimizer, TorchOptimizer):
|
||||
optimizer.zero_grad()
|
||||
elif isinstance(optimizer, PaddleOptimizer):
|
||||
optimizer.clear_grad()
|
||||
else:
|
||||
raise ValueError("Unknown optimizers type.")
|
||||
|
||||
def model_call(self, batch, fn: Callable, signature_fn: Optional[Callable]) -> Dict:
|
||||
if isinstance(batch, Dict) and not self.wo_auto_param_call:
|
||||
return auto_param_call(fn, batch, signature_fn=signature_fn)
|
||||
else:
|
||||
return fn(batch)
|
||||
|
||||
def get_model_call_fn(self, fn: str) -> Tuple:
|
||||
if hasattr(self.model, fn):
|
||||
fn = getattr(self.model, fn)
|
||||
if not callable(fn):
|
||||
raise RuntimeError(f"The `{fn}` attribute is not `Callable`.")
|
||||
logger.debug(f'Use {_get_fun_msg(fn, with_fp=False)}...')
|
||||
return fn, None
|
||||
elif fn in {"train_step", "evaluate_step"}:
|
||||
logger.debug(f'Use {_get_fun_msg(self.model.forward, with_fp=False)}...')
|
||||
return self.model, self.model.forward
|
||||
else:
|
||||
raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.")
|
||||
|
||||
def predict_step(self, batch):
|
||||
if isinstance(batch, Dict):
|
||||
return auto_param_call(self._predict_step, batch)
|
||||
else:
|
||||
return self._predict_step(batch)
|
||||
|
||||
@rank_zero_call
|
||||
def save_model(self, filepath: str, only_state_dict: bool = True, model_save_fn: Optional[Callable] = None):
|
||||
r"""
|
||||
暂时不提供保存整个模型的方法
|
||||
"""
|
||||
if only_state_dict == False:
|
||||
logger.warn("TorchPaddleModule only support saving state dicts now.")
|
||||
if model_save_fn is not None:
|
||||
model_save_fn(filepath)
|
||||
else:
|
||||
model = self.unwrap_model()
|
||||
self.move_model_to_device(model, "cpu")
|
||||
self.model.save(filepath)
|
||||
self.move_model_to_device(model, self.model_device)
|
||||
|
||||
def load_model(self, filepath: str):
|
||||
"""
|
||||
加载模型的加载函数；
|
||||
|
||||
:param filepath: 保存文件的文件位置（需要包括文件名）；
|
||||
:return:
|
||||
"""
|
||||
return self.model.load(filepath)
|
||||
|
||||
def save(self):
|
||||
...
|
||||
|
||||
def load(self):
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def move_model_to_device(model: MixModule, device: str):
|
||||
if device is not None:
|
||||
model.to(device)
|
||||
|
||||
def unwrap_model(self):
|
||||
return self.model
|
||||
|
||||
@staticmethod
|
||||
def tensor_to_numeric(tensor):
|
||||
if tensor is None:
|
||||
return None
|
||||
|
||||
def _translate(_data):
|
||||
return _data.tolist()
|
||||
|
||||
return apply_to_collection(
|
||||
data=tensor,
|
||||
dtype=(paddle.Tensor, torch.Tensor),
|
||||
function=_translate
|
||||
)
|
||||
|
||||
def set_model_mode(self, mode: str):
|
||||
assert mode in {"train", "eval"}
|
||||
getattr(self.model, mode)()
|
||||
|
||||
def get_model_device(self):
|
||||
return self.model_device
|
||||
|
||||
@property
|
||||
def data_device(self):
|
||||
if self.model_device is not None:
|
||||
return self.model_device
|
||||
else:
|
||||
return self._data_device
|
||||
|
||||
def set_model_mode(self, mode: str):
|
||||
assert mode in {"train", "eval"}
|
||||
getattr(self.model, mode)()
|
||||
|
||||
def set_sampler_epoch(self, dataloader: Union['TorchDataLoader', 'PaddleDataLoader'], cur_epoch_idx):
|
||||
# 保证 ddp 训练时的 shuffle=True 时的正确性，因为需要保证每一个进程上的 sampler 的shuffle 的随机数种子是一样的；
|
||||
return dataloader
|
||||
@ -1,4 +0,0 @@
|
||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE
|
||||
|
||||
if _NEED_IMPORT_PADDLE:
|
||||
pass
|
||||
@ -11,7 +11,6 @@ __all__ = [
|
||||
'is_in_fnlp_paddle_dist',
|
||||
'is_in_paddle_launch_dist',
|
||||
'f_rich_progress',
|
||||
'torch_paddle_move_data_to_device',
|
||||
'torch_move_data_to_device',
|
||||
'get_fn_arg_names',
|
||||
'auto_param_call',
|
||||
@ -32,7 +31,6 @@ from .jittor_utils import is_jittor_dataset, jittor_collate_wraps
|
||||
from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \
|
||||
is_in_fnlp_paddle_dist, is_in_paddle_launch_dist
|
||||
from .rich_progress import f_rich_progress
|
||||
from .torch_paddle_utils import torch_paddle_move_data_to_device
|
||||
from .torch_utils import torch_move_data_to_device
|
||||
from .utils import *
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import functools
|
||||
__all__ = []
|
||||
|
||||
class DummyClass:
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
"""
|
||||
该文件用于为fastNLP提供一个统一的progress bar管理，通过共用一个Task对象，trainer中的progress bar和evaluation中的progress bar才能
|
||||
不冲突
|
||||
|
||||
该文件用于为 ``fastNLP`` 提供一个统一的 ``progress bar`` 管理，通过共用一个``Task`` 对象， :class:`~fastNLP.core.Trainer` 中
|
||||
的 ``progress bar`` 和 :class:`~fastNLP.core.Evaluator` 中的 ``progress bar`` 才能不冲突
|
||||
"""
|
||||
import sys
|
||||
from typing import Any, Union, Optional
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
|
||||
|
||||
if _NEED_IMPORT_PADDLE:
|
||||
import paddle
|
||||
|
||||
if _NEED_IMPORT_TORCH:
|
||||
import torch
|
||||
|
||||
__all__ = [
|
||||
"torch_paddle_move_data_to_device",
|
||||
]
|
||||
|
||||
from .utils import apply_to_collection
|
||||
from .paddle_utils import paddle_to
|
||||
|
||||
|
||||
def torch_paddle_move_data_to_device(batch: Any, device: Optional[str] = None, non_blocking: Optional[bool] = True,
|
||||
data_device: Optional[str] = None) -> Any:
|
||||
|
||||
r"""
|
||||
将数据集合传输到给定设备。只有paddle.Tensor和torch.Tensor对象会被传输到设备中，其余保持不变
|
||||
|
||||
:param batch:
|
||||
:param device:
|
||||
:param non_blocking:
|
||||
:param data_device:
|
||||
:return: 相同的集合，但所有包含的张量都驻留在新设备上；
|
||||
"""
|
||||
|
||||
if device is None:
|
||||
if data_device is not None:
|
||||
device = data_device
|
||||
else:
|
||||
return batch
|
||||
|
||||
torch_device = device.replace("gpu", "cuda")
|
||||
paddle_device = device.replace("cuda", "gpu")
|
||||
|
||||
def batch_to(data: Any) -> Any:
|
||||
if isinstance(data, torch.Tensor):
|
||||
data = data.to(torch_device, non_blocking=non_blocking)
|
||||
elif isinstance(data, paddle.Tensor):
|
||||
data = paddle_to(data, paddle_device)
|
||||
|
||||
return data
|
||||
|
||||
return apply_to_collection(batch, dtype=(paddle.Tensor, torch.Tensor), function=batch_to)
|
||||
@ -10,10 +10,6 @@ from typing import Callable, List, Any, Dict, AnyStr, Union, Mapping, Sequence
|
||||
from typing import Tuple, Optional
|
||||
from time import sleep
|
||||
|
||||
try:
|
||||
from typing import Literal, Final
|
||||
except ImportError:
|
||||
from typing_extensions import Literal, Final
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
@ -22,7 +18,6 @@ import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
from fastNLP.core.log import logger
|
||||
from ...envs import SUPPORT_BACKENDS
|
||||
|
||||
|
||||
__all__ = [
|
||||
@ -43,10 +38,10 @@ __all__ = [
|
||||
|
||||
def get_fn_arg_names(fn: Callable) -> List[str]:
|
||||
r"""
|
||||
返回一个函数的所有参数的名字；
|
||||
返回一个函数所有参数的名字
|
||||
|
||||
:param fn: 需要查询的函数；
|
||||
:return: 一个列表，其中的元素则是查询函数的参数的字符串名字；
|
||||
:param fn: 需要查询的函数
|
||||
:return: 一个列表，其中的元素是函数 ``fn`` 参数的字符串名字
|
||||
"""
|
||||
return list(inspect.signature(fn).parameters)
|
||||
|
||||
@ -54,24 +49,18 @@ def get_fn_arg_names(fn: Callable) -> List[str]:
|
||||
def auto_param_call(fn: Callable, *args, signature_fn: Optional[Callable] = None,
|
||||
mapping: Optional[Dict[AnyStr, AnyStr]] = None) -> Any:
|
||||
r"""
|
||||
该函数会根据输入函数的形参名从*args（因此都需要是dict类型）中找到匹配的值进行调用，如果传入的数据与fn的形参不匹配，可以通过mapping
|
||||
参数进行转换。mapping参数中的一对（key，value）表示以这个key在*args中找到值，并将这个值传递给形参名为value的参数。
|
||||
该函数会根据输入函数的形参名从 ``*args`` （因此都需要是 ``dict`` 类型）中找到匹配的值进行调用，如果传入的数据与 ``fn`` 的形参不匹配，可以通过
|
||||
``mapping`` 参数进行转换。``mapping`` 参数中的一对 ``(key, value)`` 表示在 ``*args`` 中找到 ``key`` 对应的值，并将这个值传递给形参中名为
|
||||
``value`` 的参数。
|
||||
|
||||
1.该函数用来提供给用户根据字符串匹配从而实现自动调用；
|
||||
2.注意 mapping 默认为 None，如果你希望指定输入和运行函数的参数的对应方式，那么你应当让 mapping 为一个这样的字典传入进来；
|
||||
如果 mapping 不为 None，那么我们一定会先使用 mapping 将输入的字典的 keys 修改过来，因此请务必亲自检查 mapping 的正确性；
|
||||
3.如果输入的函数的参数有默认值，那么如果之后的输入中没有该参数对应的值，我们就会使用该参数对应的默认值，否则也会使用之后的输入的值；
|
||||
4.如果输入的函数是一个 `partial` 函数，情况同 '3.'，即和默认参数的情况相同；
|
||||
|
||||
:param fn: 用来进行实际计算的函数，其参数可以包含有默认值；
|
||||
:param args: 一系列的位置参数，应当为一系列的字典，我们需要从这些输入中提取 `fn` 计算所需要的实际参数；
|
||||
:param signature_fn: 函数，用来替换 `fn` 的函数签名，如果该参数不为 None，那么我们首先会从该函数中提取函数签名，然后通过该函数签名提取
|
||||
参数值后，再传给 `fn` 进行实际的运算；
|
||||
:param mapping: 一个字典，用来更改其前面的字典的键值；
|
||||
|
||||
:return: 返回 `fn` 运行的结果；
|
||||
1. 该函数用来提供给用户根据字符串匹配从而实现自动调用；
|
||||
2. 注意 ``mapping`` 默认为 ``None``，如果你希望指定输入和运行函数的参数的对应方式，那么你应当让 ``mapping`` 为一个字典传入进来；
|
||||
如果 ``mapping`` 不为 ``None``，那么我们一定会先使用 ``mapping`` 将输入的字典的 ``keys`` 修改过来，因此请务必亲自检查 ``mapping`` 的正确性；
|
||||
3. 如果输入的函数的参数有默认值，那么如果之后的输入中没有该参数对应的值，我们就会使用该参数对应的默认值，否则也会使用之后的输入的值；
|
||||
4. 如果输入的函数是一个 ``partial`` 函数，情况同第三点，即和默认参数的情况相同；
|
||||
|
||||
Examples::
|
||||
|
||||
>>> # 1
|
||||
>>> loss_fn = CrossEntropyLoss() # 如果其需要的参数为 def CrossEntropyLoss(y, pred)；
|
||||
>>> batch = {"x": 20, "y": 1}
|
||||
@ -84,6 +73,14 @@ def auto_param_call(fn: Callable, *args, signature_fn: Optional[Callable] = None
|
||||
>>> print(auto_param_call(test_fn, {"x": 10}, {"y": 20, "a": 30})) # res: 70
|
||||
>>> print(auto_param_call(partial(test_fn, a=100), {"x": 10}, {"y": 20})) # res: 140
|
||||
>>> print(auto_param_call(partial(test_fn, a=100), {"x": 10}, {"y": 20, "a": 200})) # res: 240
|
||||
|
||||
:param fn: 用来进行实际计算的函数，其参数可以包含有默认值；
|
||||
:param args: 一系列的位置参数，应当为一系列的字典，我们需要从这些输入中提取 ``fn`` 计算所需要的实际参数；
|
||||
:param signature_fn: 函数，用来替换 ``fn`` 的函数签名，如果该参数不为 ``None``，那么我们首先会从该函数中提取函数签名，然后通过该函数签名提取
|
||||
参数值后，再传给 ``fn`` 进行实际的运算；
|
||||
:param mapping: 一个字典，用来更改其前面的字典的键值；
|
||||
|
||||
:return: 返回 ``fn`` 运行的结果；
|
||||
"""
|
||||
|
||||
if signature_fn is not None:
|
||||
@ -226,13 +223,13 @@ def _check_valid_parameters_number(fn, expected_params:List[str], fn_name=None):
|
||||
|
||||
def check_user_specific_params(user_params: Dict, fn: Callable):
|
||||
"""
|
||||
该函数使用用户的输入来对指定函数的参数进行赋值；
|
||||
主要用于一些用户无法直接调用函数的情况；
|
||||
该函数主要的作用在于帮助检查用户对使用函数 fn 的参数输入是否有误；
|
||||
该函数使用用户的输入来对指定函数的参数进行赋值，主要用于一些用户无法直接调用函数的情况；
|
||||
该函数主要的作用在于帮助检查用户对使用函数 ``fn`` 的参数输入是否有误；
|
||||
|
||||
:param user_params: 用户指定的参数的值，应当是一个字典，其中 key 表示每一个参数的名字，value 为每一个参数应当的值；
|
||||
:param fn: 会被调用的函数；
|
||||
:return: 返回一个字典，其中为在之后调用函数 fn 时真正会被传进去的参数的值；
|
||||
:param user_params: 用户指定的参数的值，应当是一个字典，其中 ``key`` 表示每一个参数的名字，
|
||||
``value`` 为每一个参数的值；
|
||||
:param fn: 将要被调用的函数；
|
||||
:return: 返回一个字典，其中为在之后调用函数 ``fn`` 时真正会被传进去的参数的值；
|
||||
"""
|
||||
|
||||
fn_arg_names = get_fn_arg_names(fn)
|
||||
@ -243,6 +240,9 @@ def check_user_specific_params(user_params: Dict, fn: Callable):
|
||||
|
||||
|
||||
def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict:
|
||||
"""
|
||||
将传入的 `dataclass` 实例转换为字典。
|
||||
"""
|
||||
if not is_dataclass(data):
|
||||
raise TypeError(f"Parameter `data` can only be `dataclass` type instead of {type(data)}.")
|
||||
_dict = dict()
|
||||
@ -253,21 +253,31 @@ def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict:
|
||||
|
||||
def match_and_substitute_params(mapping: Optional[Union[Callable, Dict]] = None, data: Optional[Any] = None) -> Any:
|
||||
r"""
|
||||
用来实现将输入：batch，或者输出：outputs，通过 `mapping` 将键值进行更换的功能；
|
||||
该函数应用于 `input_mapping` 和 `output_mapping`；
|
||||
对于 `input_mapping`，该函数会在 `TrainBatchLoop` 中取完数据后立刻被调用；
|
||||
对于 `output_mapping`，该函数会在 `Trainer.train_step` 以及 `Evaluator.train_step` 中得到结果后立刻被调用；
|
||||
用来实现将输入的 ``batch``，或者输出的 ``outputs``，通过 ``mapping`` 将键值进行更换的功能；
|
||||
该函数应用于 ``input_mapping`` 和 ``output_mapping``；
|
||||
|
||||
对于 ``input_mapping``，该函数会在 :class:`~fastNLP.core.controllers.TrainBatchLoop` 中取完数据后立刻被调用；
|
||||
对于 ``output_mapping``，该函数会在 :class:`~fastNLP.core.Trainer` 的 :meth:`~fastNLP.core.Trainer.train_step`
|
||||
以及 :class:`~fastNLP.core.Evaluator` 的 :meth:`~fastNLP.core.Evaluator.train_step` 中得到结果后立刻被调用；
|
||||
|
||||
转换的逻辑按优先级依次为：
|
||||
|
||||
1. 如果 `mapping` 是一个函数，那么会直接返回 `mapping(data)`；
|
||||
2. 如果 `mapping` 是一个 `Dict`，那么 `data` 的类型只能为以下三种： [`Dict`, `dataclass`, `Sequence`]；
|
||||
如果 `data` 是 `Dict`，那么该函数会将 `data` 的 key 替换为 mapping[key]；
|
||||
如果 `data` 是 `dataclass`，那么该函数会先使用 `dataclasses.asdict` 函数将其转换为 `Dict`，然后进行转换；
|
||||
如果 `data` 是 `Sequence`，那么该函数会先将其转换成一个对应的 `Dict`：{"_0": list[0], "_1": list[1], ...}，然后使用
|
||||
mapping对这个 `Dict` 进行转换，如果没有匹配上mapping中的key则保持"_number"这个形式。
|
||||
1. 如果 ``mapping`` 是一个函数，那么会直接返回 ``mapping(data)``；
|
||||
2. 如果 ``mapping`` 是一个 ``Dict``，那么 ``data`` 的类型只能为以下三种： ``[Dict, dataclass, Sequence]``；
|
||||
|
||||
* 如果 ``data`` 是 ``Dict``，那么该函数会将 ``data`` 的 ``key`` 替换为 ``mapping[key]``；
|
||||
* 如果 ``data`` 是 ``dataclass``，那么该函数会先使用 :func:`dataclasses.asdict` 函数将其转换为 ``Dict``，然后进行转换；
|
||||
* 如果 ``data`` 是 ``Sequence``，那么该函数会先将其转换成一个对应的字典::
|
||||
|
||||
{
|
||||
"_0": list[0],
|
||||
"_1": list[1],
|
||||
...
|
||||
}
|
||||
|
||||
:param mapping: 用于转换的字典或者函数；mapping是函数时，返回值必须为字典类型。
|
||||
然后使用 ``mapping`` 对这个 ``Dict`` 进行转换，如果没有匹配上 ``mapping`` 中的 ``key`` 则保持 ``\'\_number\'`` 这个形式。
|
||||
|
||||
:param mapping: 用于转换的字典或者函数；``mapping`` 是函数时，返回值必须为字典类型。
|
||||
:param data: 需要被转换的对象；
|
||||
:return: 返回转换好的结果；
|
||||
"""
|
||||
@ -320,21 +330,20 @@ def apply_to_collection(
|
||||
include_none: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""将函数 function 递归地在 data 中的元素执行，但是仅在满足元素为 dtype 时执行。
|
||||
"""
|
||||
使用函数 ``function`` 递归地在 ``data`` 中的元素执行，但是仅在满足元素为 ``dtype`` 时执行。
|
||||
|
||||
this function credit to: https://github.com/PyTorchLightning/pytorch-lightning
|
||||
Args:
|
||||
data: the collection to apply the function to
|
||||
dtype: the given function will be applied to all elements of this dtype
|
||||
function: the function to apply
|
||||
*args: positional arguments (will be forwarded to calls of ``function``)
|
||||
wrong_dtype: the given function won't be applied if this type is specified and the given collections
|
||||
is of the ``wrong_dtype`` even if it is of type ``dtype``
|
||||
include_none: Whether to include an element if the output of ``function`` is ``None``.
|
||||
**kwargs: keyword arguments (will be forwarded to calls of ``function``)
|
||||
该函数参考了 `pytorch-lightning <https://github.com/PyTorchLightning/pytorch-lightning>`_ 的实现
|
||||
|
||||
Returns:
|
||||
The resulting collection
|
||||
:param data: 需要进行处理的数据集合或数据
|
||||
:param dtype: 数据的类型，函数 ``function`` 只会被应用于 ``data`` 中类型为 ``dtype`` 的数据
|
||||
:param function: 对数据进行处理的函数
|
||||
:param args: ``function`` 所需要的其它参数
|
||||
:param wrong_dtype: ``function`` 一定不会生效的数据类型。如果数据既是 ``wrong_dtype`` 类型又是 ``dtype`` 类型
|
||||
那么也不会生效。
|
||||
:param include_none: 是否包含执行结果为 ``None`` 的数据，默认为 ``True``。
|
||||
:param kwargs: ``function`` 所需要的其它参数
|
||||
:return: 经过 ``function`` 处理后的数据集合
|
||||
"""
|
||||
# Breaking condition
|
||||
if isinstance(data, dtype) and (wrong_dtype is None or not isinstance(data, wrong_dtype)):
|
||||
@ -402,16 +411,18 @@ def apply_to_collection(
|
||||
@contextmanager
|
||||
def nullcontext():
|
||||
r"""
|
||||
用来实现一个什么 dummy 的 context 上下文环境；
|
||||
实现一个什么都不做的上下文环境
|
||||
"""
|
||||
yield
|
||||
|
||||
|
||||
def sub_column(string: str, c: int, c_size: int, title: str) -> str:
|
||||
r"""
|
||||
对传入的字符串进行截断，方便在命令行中显示
|
||||
|
||||
:param string: 要被截断的字符串
|
||||
:param c: 命令行列数
|
||||
:param c_size: instance或dataset field数
|
||||
:param c_size: :class:`~fastNLP.core.Instance` 或 :class:`fastNLP.core.DataSet` 的 ``field`` 数目
|
||||
:param title: 列名
|
||||
:return: 对一个过长的列进行截断的结果
|
||||
"""
|
||||
@ -442,18 +453,17 @@ def _is_iterable(value):
|
||||
|
||||
def pretty_table_printer(dataset_or_ins) -> PrettyTable:
|
||||
r"""
|
||||
:param dataset_or_ins: 传入一个dataSet或者instance
|
||||
在 ``fastNLP`` 中展示数据的函数::
|
||||
|
||||
.. code-block::
|
||||
|
||||
ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"])
|
||||
>>> ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"])
|
||||
+-----------+-----------+-----------------+
|
||||
| field_1 | field_2 | field_3 |
|
||||
+-----------+-----------+-----------------+
|
||||
| [1, 1, 1] | [2, 2, 2] | ['a', 'b', 'c'] |
|
||||
+-----------+-----------+-----------------+
|
||||
|
||||
:return: 以 pretty table的形式返回根据terminal大小进行自动截断
|
||||
:param dataset_or_ins: 要展示的 :class:`~fastNLP.core.DataSet` 或者 :class:`~fastNLP.core.Instance`
|
||||
:return: 根据 ``terminal`` 大小进行自动截断的数据表格
|
||||
"""
|
||||
x = PrettyTable()
|
||||
try:
|
||||
@ -486,7 +496,7 @@ def pretty_table_printer(dataset_or_ins) -> PrettyTable:
|
||||
|
||||
|
||||
class Option(dict):
|
||||
r"""a dict can treat keys as attributes"""
|
||||
r"""将键转化为属性的字典类型"""
|
||||
|
||||
def __getattr__(self, item):
|
||||
try:
|
||||
@ -516,11 +526,10 @@ _emitted_deprecation_warnings = set()
|
||||
|
||||
|
||||
def deprecated(help_message: Optional[str] = None):
|
||||
"""Decorator to mark a function as deprecated.
|
||||
"""
|
||||
标记当前功能已经过时的装饰器。
|
||||
|
||||
Args:
|
||||
help_message (`Optional[str]`): An optional message to guide the user on how to
|
||||
switch to non-deprecated usage of the library.
|
||||
:param help_message: 一段指引信息，告知用户如何将代码切换为当前版本提倡的用法。
|
||||
"""
|
||||
|
||||
def decorator(deprecated_function: Callable):
|
||||
@ -549,11 +558,10 @@ def deprecated(help_message: Optional[str] = None):
|
||||
return decorator
|
||||
|
||||
|
||||
def seq_len_to_mask(seq_len, max_len=None):
|
||||
def seq_len_to_mask(seq_len, max_len: Optional[int]):
|
||||
r"""
|
||||
|
||||
将一个表示sequence length的一维数组转换为二维的mask，不包含的位置为0。
|
||||
转变 1-d seq_len到2-d mask.
|
||||
将一个表示 ``sequence length`` 的一维数组转换为二维的 ``mask`` ，不包含的位置为 **0**。
|
||||
|
||||
.. code-block::
|
||||
|
||||
@ -570,10 +578,11 @@ def seq_len_to_mask(seq_len, max_len=None):
|
||||
>>>print(mask.size())
|
||||
torch.Size([14, 100])
|
||||
|
||||
:param np.ndarray,torch.LongTensor seq_len: shape将是(B,)
|
||||
:param int max_len: 将长度pad到这个长度。默认(None)使用的是seq_len中最长的长度。但在nn.DataParallel的场景下可能不同卡的seq_len会有
|
||||
区别，所以需要传入一个max_len使得mask的长度是pad到该长度。
|
||||
:return: np.ndarray, torch.Tensor 。shape将是(B, max_length)， 元素类似为bool或torch.uint8
|
||||
:param seq_len: 大小为是 ``(B,)`` 的长度序列
|
||||
:param int max_len: 将长度 ``pad`` 到 ``max_len``。默认情况（为 ``None``）使用的是 ``seq_len`` 中最长的长度。
|
||||
但在 :class:`torch.nn.DataParallel` 等分布式的场景下可能不同卡的 ``seq_len`` 会有区别，所以需要传入
|
||||
一个 ``max_len`` 使得 ``mask`` 的长度 ``pad`` 到该长度。
|
||||
:return: 大小为 ``(B, max_len)`` 的 ``mask``， 元素类型为 ``bool`` 或 ``uint8``
|
||||
"""
|
||||
if isinstance(seq_len, np.ndarray):
|
||||
assert len(np.shape(seq_len)) == 1, f"seq_len can only have one dimension, got {len(np.shape(seq_len))}."
|
||||
|
||||
@ -6,6 +6,7 @@ from packaging.version import Version
|
||||
import subprocess
|
||||
import pkg_resources
|
||||
|
||||
__all__ = []
|
||||
|
||||
def _module_available(module_path: str) -> bool:
|
||||
"""Check if a path is available in your environment.
|
||||
@ -48,10 +49,11 @@ def _compare_version(package: str, op: Callable, version: str, use_base_version:
|
||||
pkg_version = Version(pkg_version.base_version)
|
||||
return op(pkg_version, Version(version))
|
||||
|
||||
def get_gpu_count():
|
||||
def get_gpu_count() -> int:
|
||||
"""
|
||||
利用命令行获取gpu数目的函数
|
||||
:return: gpu数目，如果没有显卡设备则为-1
|
||||
利用命令行获取 ``gpu`` 数目的函数
|
||||
|
||||
:return: 显卡数目，如果没有显卡设备则为-1
|
||||
"""
|
||||
try:
|
||||
lines = subprocess.check_output(['nvidia-smi', '--query-gpu=memory.used', '--format=csv'])
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
__all__ = [
|
||||
"MixModule",
|
||||
"torch2paddle",
|
||||
"paddle2torch",
|
||||
"torch2jittor",
|
||||
"jittor2torch",
|
||||
]
|
||||
|
||||
from .mix_modules import MixModule, torch2paddle, paddle2torch, torch2jittor, jittor2torch
|
||||
@ -1,10 +0,0 @@
|
||||
__all__ = [
|
||||
"MixModule",
|
||||
"torch2paddle",
|
||||
"paddle2torch",
|
||||
"torch2jittor",
|
||||
"jittor2torch",
|
||||
]
|
||||
|
||||
from .mix_module import MixModule
|
||||
from .utils import *
|
||||
@ -1,310 +0,0 @@
|
||||
import os
|
||||
import io
|
||||
import pickle
|
||||
from typing import Dict
|
||||
from collections import OrderedDict
|
||||
|
||||
import numpy as np
|
||||
|
||||
from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
|
||||
from fastNLP.core.utils.paddle_utils import paddle_to
|
||||
|
||||
if _NEED_IMPORT_PADDLE:
|
||||
import paddle
|
||||
from paddle.nn import Layer as PaddleLayer
|
||||
|
||||
if _NEED_IMPORT_TORCH:
|
||||
import torch
|
||||
from torch.nn import Module as TorchModule, Parameter as TorchParameter
|
||||
|
||||
if _NEED_IMPORT_JITTOR:
|
||||
import jittor
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MixModule",
|
||||
]
|
||||
|
||||
class MixModule:
|
||||
"""
|
||||
TODO: 支持不同的混合方式；添加state_dict的支持；如果参数里有List of Tensors该怎么处理；
|
||||
是否需要仿照Module那样在初始化的时候给各种模型分类
|
||||
可以同时使用Torch和Paddle框架的混合模型
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.forward(*args, **kwargs)
|
||||
|
||||
def named_parameters(self, prefix='', recurse: bool=True, backend=None):
|
||||
"""
|
||||
返回模型的名字和参数
|
||||
|
||||
:param prefix: 输出时在参数名前加上的前缀
|
||||
:param recurse: 是否递归地输出参数
|
||||
:param backend: `backend`=`None`时，将所有模型和张量的参数返回；
|
||||
`backend`=`torch`时，返回`torch`的参数；
|
||||
`backend`=`paddle`时，返回`paddle`的参数。
|
||||
"""
|
||||
if backend is None:
|
||||
generator = self.attributes(TorchModule, TorchParameter, PaddleLayer)
|
||||
elif backend == "torch":
|
||||
generator = self.attributes(TorchModule, TorchParameter)
|
||||
elif backend == "paddle":
|
||||
generator = self.attributes(PaddleLayer)
|
||||
else:
|
||||
raise ValueError("Unknown backend parameter.")
|
||||
|
||||
for name, value in generator:
|
||||
name = prefix + ('.' if prefix else '') + name
|
||||
if isinstance(value, TorchParameter):
|
||||
# 非Module/Layer类型，直接输出名字和值
|
||||
yield name, value
|
||||
elif recurse:
|
||||
# 递归地调用named_parameters
|
||||
for name_r, value_r in value.named_parameters(name, recurse):
|
||||
yield name_r, value_r
|
||||
|
||||
def parameters(self, recurse: bool = True, backend: str = None):
|
||||
"""
|
||||
返回模型的参数
|
||||
|
||||
:param recurse:
|
||||
:param backend: `backend`=`None`时，将所有模型和张量的参数返回；
|
||||
`backend`=`torch`时，返回`torch`的参数；
|
||||
`backend`=`paddle`时，返回`paddle`的参数。
|
||||
"""
|
||||
for name, value in self.named_parameters(recurse=recurse, backend=backend):
|
||||
yield value
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def train_step(self, batch):
|
||||
raise NotImplementedError
|
||||
|
||||
def test_step(self, batch):
|
||||
raise NotImplementedError
|
||||
|
||||
def evaluate_step(self, batch):
|
||||
raise NotImplementedError
|
||||
|
||||
def train(self):
|
||||
for name, value in self.attributes(TorchModule, PaddleLayer):
|
||||
value.train()
|
||||
|
||||
def eval(self):
|
||||
for name, value in self.attributes(TorchModule, PaddleLayer):
|
||||
value.eval()
|
||||
|
||||
def to(self, device):
|
||||
"""
|
||||
:param device: 设备名
|
||||
"""
|
||||
# 有jittor的话 warning
|
||||
if device == "cpu":
|
||||
paddle_device = device
|
||||
elif device.startswith("cuda"):
|
||||
paddle_device = device.replace("cuda", "gpu")
|
||||
elif device.startswith("gpu"):
|
||||
paddle_device = device
|
||||
device = device.replace("gpu", "cuda")
|
||||
else:
|
||||
raise ValueError("Device value error")
|
||||
|
||||
for name, value in self.attributes(TorchModule):
|
||||
# torch的to函数不影响Tensor
|
||||
vars(self)[name] = value.to(device)
|
||||
for name, value in self.attributes(TorchParameter):
|
||||
# Parameter在经过to函数后会变成Tensor类型
|
||||
vars(self)[name] = TorchParameter(value.to(device), requires_grad=value.requires_grad)
|
||||
|
||||
for name, value in self.attributes(PaddleLayer):
|
||||
vars(self)[name] = value.to(paddle_device)
|
||||
for name, value in self.attributes(paddle.Tensor):
|
||||
# paddle的to函数会影响到Tensor
|
||||
vars(self)[name] = paddle_to(value, paddle_device)
|
||||
|
||||
return self
|
||||
|
||||
def state_dict(self, backend: str = None) -> Dict:
|
||||
"""
|
||||
返回模型的state_dict。
|
||||
|
||||
.. note:: torch的destination参数会在将来删除，因此不提供destination参数
|
||||
|
||||
:param backend: `backend`=`None`时，将所有模型和张量的state dict返回；
|
||||
`backend`=`torch`时，返回`torch`的state dict；
|
||||
`backend`=`paddle`时，返回`paddle`的state dict。
|
||||
"""
|
||||
if backend is None:
|
||||
generator = self.attributes(TorchModule, TorchParameter, PaddleLayer)
|
||||
elif backend == "torch":
|
||||
generator = self.attributes(TorchModule, TorchParameter)
|
||||
elif backend == "paddle":
|
||||
generator = self.attributes(PaddleLayer)
|
||||
else:
|
||||
raise ValueError(f"Unknown backend {backend}.")
|
||||
|
||||
destination = OrderedDict()
|
||||
|
||||
for name, value in generator:
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, TorchParameter):
|
||||
destination[name] = value
|
||||
else:
|
||||
# 不同框架state_dict函数的参数名和顺序不同
|
||||
if isinstance(value, PaddleLayer):
|
||||
kwargs = {
|
||||
"structured_name_prefix": name + ".",
|
||||
}
|
||||
elif isinstance(value, TorchModule):
|
||||
kwargs = {
|
||||
"prefix": name + ".",
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"Unknown item type {type(value)}")
|
||||
destination.update(value.state_dict(**kwargs))
|
||||
|
||||
return destination
|
||||
|
||||
def save_state_dict_to_file(self, path: str):
|
||||
"""
|
||||
保存模型的state dict到path
|
||||
"""
|
||||
# TODO 设备限制
|
||||
filename = os.path.basename(path)
|
||||
if filename == "":
|
||||
raise ValueError("Received empty filename.")
|
||||
dirname = os.path.dirname(path)
|
||||
if dirname and not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
protocol = 4
|
||||
|
||||
saved = {}
|
||||
paddle_dict = self.state_dict(backend="paddle")
|
||||
torch_dict = self.state_dict(backend="torch")
|
||||
# 保存paddle部分
|
||||
# 调用paddle保存时的处理函数
|
||||
paddle_saved_obj = paddle.framework.io._build_saved_state_dict(paddle_dict)
|
||||
paddle_saved_obj = paddle.fluid.io._unpack_saved_dict(paddle_saved_obj, protocol)
|
||||
# 将返回的dict保存
|
||||
saved["paddle"] = paddle_saved_obj
|
||||
|
||||
# 保存torch部分
|
||||
buffer = io.BytesIO()
|
||||
torch.save(torch_dict, buffer)
|
||||
saved["torch"] = buffer.getvalue()
|
||||
|
||||
# 保存
|
||||
with open(path, "wb") as f:
|
||||
pickle.dump(saved, f, protocol)
|
||||
|
||||
def load_state_dict_from_file(self, path: str):
|
||||
"""
|
||||
从 `path` 中加载保存的state dict
|
||||
"""
|
||||
state_dict = {}
|
||||
with open(path, "rb") as f:
|
||||
loaded = pickle.load(f)
|
||||
# 加载paddle的数据
|
||||
paddle_loaded_obj = loaded["paddle"]
|
||||
paddle_load_result = paddle.fluid.io._pack_loaded_dict(paddle_loaded_obj)
|
||||
if "StructuredToParameterName@@" in paddle_load_result:
|
||||
for key in paddle_load_result["StructuredToParameterName@@"]:
|
||||
if isinstance(paddle_load_result[key], np.ndarray):
|
||||
paddle_load_result[key] = paddle.to_tensor(paddle_load_result[key])
|
||||
state_dict.update(paddle_load_result)
|
||||
# 加载torch的数据
|
||||
torch_loaded_obj = loaded["torch"]
|
||||
torch_bytes = io.BytesIO(torch_loaded_obj)
|
||||
torch_load_result = torch.load(torch_bytes)
|
||||
state_dict.update(torch_load_result)
|
||||
|
||||
self.load_state_dict(state_dict)
|
||||
|
||||
def load_state_dict(self, state_dict):
|
||||
"""
|
||||
从state dict中加载数据
|
||||
"""
|
||||
missing_keys = []
|
||||
unexpected_keys = []
|
||||
error_msgs = []
|
||||
new_state = {}
|
||||
|
||||
local_state = self.state_dict()
|
||||
|
||||
# 对字典内容按前缀进行归类
|
||||
for key, value in state_dict.items():
|
||||
splited = key.split(".", 1)
|
||||
if len(splited) == 1:
|
||||
# 没有前缀，实际上只有torch.nn.Parameter会进入这种情况
|
||||
new_state[key] = value
|
||||
else:
|
||||
prefix, name = splited
|
||||
if prefix not in new_state:
|
||||
new_state[prefix] = {}
|
||||
new_state[prefix][name] = value
|
||||
|
||||
for key, param in self.attributes(TorchModule, TorchParameter, PaddleLayer):
|
||||
if key in new_state:
|
||||
# 在传入的字典中找到了对应的值
|
||||
input_param = new_state[key]
|
||||
if not isinstance(input_param, dict):
|
||||
# 且不是字典，即上述没有前缀的情况
|
||||
# 按照torch.nn.Module._load_from_state_dict进行赋值
|
||||
if not torch.overrides.is_tensor_like(input_param):
|
||||
error_msgs.append('While copying the parameter named "{}", '
|
||||
'expected torch.Tensor or Tensor-like object from checkpoint but '
|
||||
'received {}'
|
||||
.format(key, type(input_param)))
|
||||
continue
|
||||
|
||||
# This is used to avoid copying uninitialized parameters into
|
||||
# non-lazy modules, since they dont have the hook to do the checks
|
||||
# in such case, it will error when accessing the .shape attribute.
|
||||
is_param_lazy = torch.nn.parameter.is_lazy(param)
|
||||
# Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+
|
||||
if not is_param_lazy and len(param.shape) == 0 and len(input_param.shape) == 1:
|
||||
input_param = input_param[0]
|
||||
|
||||
if not is_param_lazy and input_param.shape != param.shape:
|
||||
# local shape should match the one in checkpoint
|
||||
error_msgs.append('size mismatch for {}: copying a param with shape {} from checkpoint, '
|
||||
'the shape in current model is {}.'
|
||||
.format(key, input_param.shape, param.shape))
|
||||
continue
|
||||
try:
|
||||
with torch.no_grad():
|
||||
param.copy_(input_param)
|
||||
except Exception as ex:
|
||||
error_msgs.append('While copying the parameter named "{}", '
|
||||
'whose dimensions in the model are {} and '
|
||||
'whose dimensions in the checkpoint are {}, '
|
||||
'an exception occurred : {}.'
|
||||
.format(key, param.size(), input_param.size(), ex.args))
|
||||
else:
|
||||
# 否则在子模块中
|
||||
if isinstance(param, TorchModule):
|
||||
# torch模块
|
||||
# 由于paddle没有提供类似strict的参数，因此也不对torch作要求
|
||||
param.load_state_dict(input_param, strict=False)
|
||||
elif isinstance(param, PaddleLayer):
|
||||
# paddle模块
|
||||
param.load_dict(input_param)
|
||||
else:
|
||||
missing_keys.append(key)
|
||||
|
||||
if len(error_msgs) > 0:
|
||||
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
|
||||
self.__class__.__name__, "\n\t".join(error_msgs)))
|
||||
|
||||
def attributes(self, *types):
|
||||
"""
|
||||
查找对应类型的成员
|
||||
"""
|
||||
for name, value in vars(self).items():
|
||||
if isinstance(value, types):
|
||||
yield name, value
|
||||
@ -1,233 +0,0 @@
|
||||
import warnings
|
||||
import os
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from fastNLP.core.utils.utils import apply_to_collection
|
||||
from fastNLP.core.utils.paddle_utils import paddle_to
|
||||
from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_TORCH, _NEED_IMPORT_PADDLE
|
||||
|
||||
if _NEED_IMPORT_PADDLE:
|
||||
import paddle
|
||||
|
||||
if _NEED_IMPORT_JITTOR:
|
||||
import jittor
|
||||
|
||||
if _NEED_IMPORT_TORCH:
|
||||
import torch
|
||||
|
||||
__all__ = [
|
||||
"paddle2torch",
|
||||
"torch2paddle",
|
||||
"jittor2torch",
|
||||
"torch2jittor",
|
||||
]
|
||||
|
||||
def _paddle2torch(paddle_tensor: 'paddle.Tensor', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor':
|
||||
"""
|
||||
将paddle tensor转换为torch tensor，并且能够保留梯度进行反向传播
|
||||
:param paddle_tensor: 要转换的paddle张量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的torch张量
|
||||
"""
|
||||
no_gradient = paddle_tensor.stop_gradient if no_gradient is None else no_gradient
|
||||
paddle_numpy = paddle_tensor.numpy()
|
||||
if not np.issubdtype(paddle_numpy.dtype, np.inexact):
|
||||
no_gradient = True
|
||||
|
||||
if target_device is None:
|
||||
if paddle_tensor.place.is_gpu_place():
|
||||
# paddlepaddle有两种Place，对应不同的device id获取方式
|
||||
if hasattr(paddle_tensor.place, "gpu_device_id"):
|
||||
# paddle.fluid.core_avx.Place
|
||||
# 在gpu环境下创建张量的话，张量的place是这一类型
|
||||
target_device = f"cuda:{paddle_tensor.place.gpu_device_id()}"
|
||||
else:
|
||||
# paddle.CUDAPlace
|
||||
target_device = f"cuda:{paddle_tensor.place.get_device_id()}"
|
||||
else:
|
||||
# TODO: 可能需要支持xpu等设备
|
||||
target_device = "cpu"
|
||||
|
||||
if not no_gradient:
|
||||
# 保持梯度，并保持反向传播
|
||||
# torch.tensor会保留numpy数组的类型
|
||||
torch_tensor = torch.tensor(paddle_numpy, requires_grad=True, device=target_device)
|
||||
hook = torch_tensor.register_hook(
|
||||
lambda grad: paddle.autograd.backward(paddle_tensor, paddle.to_tensor(grad.cpu().numpy()))
|
||||
)
|
||||
else:
|
||||
# 不保留梯度
|
||||
torch_tensor = torch.tensor(paddle_numpy, requires_grad=False, device=target_device)
|
||||
|
||||
return torch_tensor
|
||||
|
||||
|
||||
def _torch2paddle(torch_tensor: 'torch.Tensor', target_device: str = None, no_gradient: bool = None) -> 'paddle.Tensor':
|
||||
"""
|
||||
将torch tensor转换为paddle tensor，并且能够保留梯度进行反向传播。
|
||||
:param torch_tensor: 要转换的torch张量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的paddle张量
|
||||
"""
|
||||
no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient
|
||||
if target_device is None:
|
||||
if torch_tensor.is_cuda:
|
||||
target_device = f"gpu:{torch_tensor.device.index}"
|
||||
else:
|
||||
target_device = "cpu"
|
||||
|
||||
if not no_gradient:
|
||||
# 保持梯度并保持反向传播
|
||||
# paddle的stop_gradient和torch的requires_grad表现是相反的
|
||||
paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=False)
|
||||
hook = paddle_tensor.register_hook(
|
||||
lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy()))
|
||||
)
|
||||
else:
|
||||
paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=True)
|
||||
|
||||
paddle_tensor = paddle_to(paddle_tensor, target_device)
|
||||
|
||||
return paddle_tensor
|
||||
|
||||
|
||||
def _jittor2torch(jittor_var: 'jittor.Var', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor':
|
||||
"""
|
||||
将jittor Var转换为torch tensor，并且能够保留梯度进行反向传播
|
||||
:param jittor_var: 要转换的jittor变量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，根据jittor.flags.use_cuda决定。
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的torch张量
|
||||
"""
|
||||
# TODO: warning：无法保留梯度
|
||||
# jittor的grad可以通过callback进行传递
|
||||
# 如果outputs有_grad键，可以实现求导
|
||||
no_gradient = not jittor_var.requires_grad if no_gradient is None else no_gradient
|
||||
if no_gradient == False:
|
||||
warnings.warn("The result tensor will not keep gradients due to differences between jittor and pytorch.")
|
||||
jittor_numpy = jittor_var.numpy()
|
||||
if not np.issubdtype(jittor_numpy.dtype, np.inexact):
|
||||
no_gradient = True
|
||||
|
||||
if target_device is None:
|
||||
# jittor的设备分配是自动的
|
||||
# 根据use_cuda判断
|
||||
if jittor.flags.use_cuda:
|
||||
target_device = "cuda:0"
|
||||
else:
|
||||
target_device = "cpu"
|
||||
|
||||
torch_tensor = torch.tensor(jittor_numpy, requires_grad=not no_gradient, device=target_device)
|
||||
|
||||
return torch_tensor
|
||||
|
||||
|
||||
def _torch2jittor(torch_tensor: 'torch.Tensor', no_gradient: bool = None) -> 'jittor.Var':
|
||||
"""
|
||||
将torch tensor转换为jittor Var，并且能够保留梯度进行反向传播
|
||||
:param torch_tensor: 要转换的torch张量
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的jittor变量
|
||||
"""
|
||||
no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient
|
||||
|
||||
if not no_gradient:
|
||||
# 保持梯度并保持反向传播
|
||||
jittor_var = jittor.Var(torch_tensor.detach().numpy())
|
||||
jittor_var.requires_grad = True
|
||||
hook = jittor_var.register_hook(
|
||||
lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy()))
|
||||
)
|
||||
else:
|
||||
jittor_var = jittor.Var(torch_tensor.detach().numpy())
|
||||
jittor_var.requires_grad = False
|
||||
|
||||
return jittor_var
|
||||
|
||||
|
||||
def torch2paddle(torch_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
|
||||
"""
|
||||
递归地将输入中包含的torch张量转换为paddle张量
|
||||
|
||||
:param torch_in: 要转换的包含torch.Tensor类型的变量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，
|
||||
输入为`None`时，和输入的张量相同，
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 将所有torch.Tensor转换为paddle.Tensor的张量
|
||||
"""
|
||||
|
||||
return apply_to_collection(
|
||||
torch_in,
|
||||
dtype=torch.Tensor,
|
||||
function=_torch2paddle,
|
||||
target_device=target_device,
|
||||
no_gradient=no_gradient,
|
||||
)
|
||||
|
||||
|
||||
def paddle2torch(paddle_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
|
||||
"""
|
||||
递归地将输入中包含的paddle张量转换为torch张量
|
||||
|
||||
:param torch_in: 要转换的包含paddle.Tensor类型的变量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，
|
||||
输入为`None`时，和输入的张量相同，
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 将所有paddle.Tensor转换为torch.Tensor后的变量
|
||||
"""
|
||||
|
||||
return apply_to_collection(
|
||||
paddle_in,
|
||||
dtype=paddle.Tensor,
|
||||
function=_paddle2torch,
|
||||
target_device=target_device,
|
||||
no_gradient=no_gradient,
|
||||
)
|
||||
|
||||
|
||||
def jittor2torch(jittor_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
|
||||
"""
|
||||
递归地将输入中包含的jittor变量转换为torch张量
|
||||
|
||||
:param jittor_in: 要转换的jittor变量
|
||||
:param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，默认为cuda:0。
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的torch张量
|
||||
"""
|
||||
|
||||
return apply_to_collection(
|
||||
jittor_in,
|
||||
dtype=jittor.Var,
|
||||
function=_jittor2torch,
|
||||
target_device=target_device,
|
||||
no_gradient=no_gradient,
|
||||
)
|
||||
|
||||
|
||||
def torch2jittor(torch_in: Any, no_gradient: bool = None) -> Any:
|
||||
"""
|
||||
递归地将输入中包含的torch张量转换为jittor变量
|
||||
|
||||
:param torch_tensor: 要转换的torch张量
|
||||
:param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
|
||||
为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
|
||||
:return: 转换后的jittor变量
|
||||
"""
|
||||
|
||||
return apply_to_collection(
|
||||
torch_in,
|
||||
dtype=torch.Tensor,
|
||||
function=_torch2jittor,
|
||||
no_gradient=no_gradient,
|
||||
)
|
||||
@ -11,6 +11,9 @@ if _NEED_IMPORT_JITTOR:
|
||||
import jittor as jt
|
||||
from jittor import nn, Module
|
||||
from jittor.dataset import Dataset
|
||||
else:
|
||||
from fastNLP.core.utils.dummy_class import DummyClass as Module
|
||||
from fastNLP.core.utils.dummy_class import DummyClass as Dataset
|
||||
|
||||
|
||||
class JittorNormalModel_Classification(Module):
|
||||
@ -68,6 +71,7 @@ class TrainJittorConfig:
|
||||
|
||||
@pytest.mark.parametrize("driver,device", [("jittor", None)])
|
||||
@pytest.mark.parametrize("callbacks", [[RichCallback(100)]])
|
||||
@pytest.mark.jittor
|
||||
def test_trainer_jittor(
|
||||
driver,
|
||||
device,
|
||||
|
||||
@ -1,122 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from fastNLP.modules.mix_modules.mix_module import MixModule
|
||||
from fastNLP.core.drivers.torch_paddle_driver.torch_paddle_driver import TorchPaddleDriver
|
||||
from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle
|
||||
|
||||
import torch
|
||||
import paddle
|
||||
from paddle.io import Dataset, DataLoader
|
||||
import numpy as np
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试在MNIST数据集上的表现
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class MNISTDataset(Dataset):
|
||||
def __init__(self, dataset):
|
||||
|
||||
self.dataset = [
|
||||
(
|
||||
np.array(img).astype('float32').reshape(-1),
|
||||
label
|
||||
) for img, label in dataset
|
||||
]
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.dataset[idx]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
class MixMNISTModel(MixModule):
|
||||
def __init__(self):
|
||||
super(MixMNISTModel, self).__init__()
|
||||
|
||||
self.fc1 = paddle.nn.Linear(784, 64)
|
||||
self.fc2 = paddle.nn.Linear(64, 32)
|
||||
self.fc3 = torch.nn.Linear(32, 10)
|
||||
self.fc4 = torch.nn.Linear(10, 10)
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
paddle_out = self.fc1(x)
|
||||
paddle_out = self.fc2(paddle_out)
|
||||
torch_in = paddle2torch(paddle_out)
|
||||
torch_out = self.fc3(torch_in)
|
||||
torch_out = self.fc4(torch_out)
|
||||
|
||||
return torch_out
|
||||
|
||||
def train_step(self, x):
|
||||
return self.forward(x)
|
||||
|
||||
def test_step(self, x):
|
||||
return self.forward(x)
|
||||
|
||||
@pytest.mark.torchpaddle
|
||||
class TestMNIST:
|
||||
|
||||
@classmethod
|
||||
def setup_class(self):
|
||||
|
||||
self.train_dataset = paddle.vision.datasets.MNIST(mode='train')
|
||||
self.test_dataset = paddle.vision.datasets.MNIST(mode='test')
|
||||
self.train_dataset = MNISTDataset(self.train_dataset)
|
||||
|
||||
self.lr = 0.0003
|
||||
self.epochs = 20
|
||||
|
||||
self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True)
|
||||
|
||||
def setup_method(self):
|
||||
|
||||
model = MixMNISTModel()
|
||||
self.torch_loss_func = torch.nn.CrossEntropyLoss()
|
||||
|
||||
torch_opt = torch.optim.Adam(model.parameters(backend="torch"), self.lr)
|
||||
paddle_opt = paddle.optimizer.Adam(parameters=model.parameters(backend="paddle"), learning_rate=self.lr)
|
||||
|
||||
self.driver = TorchPaddleDriver(model=model, device="cuda:0")
|
||||
self.driver.set_optimizers([torch_opt, paddle_opt])
|
||||
|
||||
def test_case1(self):
|
||||
|
||||
epochs = 20
|
||||
|
||||
self.driver.setup()
|
||||
self.driver.zero_grad()
|
||||
# 开始训练
|
||||
current_epoch_idx = 0
|
||||
while current_epoch_idx < epochs:
|
||||
epoch_loss, batch = 0, 0
|
||||
self.driver.set_model_mode("train")
|
||||
self.driver.set_sampler_epoch(self.dataloader, current_epoch_idx)
|
||||
for batch, (img, label) in enumerate(self.dataloader):
|
||||
img = paddle.to_tensor(img).cuda()
|
||||
torch_out = self.driver.train_step(img)
|
||||
label = torch.from_numpy(label.numpy()).reshape(-1)
|
||||
loss = self.torch_loss_func(torch_out.cpu(), label)
|
||||
epoch_loss += loss.item()
|
||||
|
||||
self.driver.backward(loss)
|
||||
self.driver.step()
|
||||
self.driver.zero_grad()
|
||||
|
||||
current_epoch_idx += 1
|
||||
|
||||
# 开始测试
|
||||
correct = 0
|
||||
for img, label in self.test_dataset:
|
||||
|
||||
img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1))
|
||||
torch_out = self.driver.test_step(img)
|
||||
res = torch_out.softmax(-1).argmax().item()
|
||||
label = label.item()
|
||||
if res == label:
|
||||
correct += 1
|
||||
|
||||
acc = correct / len(self.test_dataset)
|
||||
assert acc > 0.85
|
||||
@ -1,204 +0,0 @@
|
||||
import paddle
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from fastNLP.core.utils.torch_paddle_utils import torch_paddle_move_data_to_device
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试将参数中包含的所有torch和paddle张量迁移到指定设备
|
||||
#
|
||||
############################################################################
|
||||
|
||||
@pytest.mark.torchpaddle
|
||||
class TestTorchPaddleMoveDataToDevice:
|
||||
|
||||
def check_gpu(self, tensor, idx):
|
||||
"""
|
||||
检查张量是否在指定显卡上的工具函数
|
||||
"""
|
||||
|
||||
if isinstance(tensor, paddle.Tensor):
|
||||
assert tensor.place.is_gpu_place()
|
||||
assert tensor.place.gpu_device_id() == idx
|
||||
elif isinstance(tensor, torch.Tensor):
|
||||
assert tensor.is_cuda
|
||||
assert tensor.device.index == idx
|
||||
|
||||
def check_cpu(self, tensor):
|
||||
if isinstance(tensor, paddle.Tensor):
|
||||
assert tensor.place.is_cpu_place()
|
||||
elif isinstance(tensor, torch.Tensor):
|
||||
assert not tensor.is_cuda
|
||||
|
||||
def test_tensor_transfer(self):
|
||||
"""
|
||||
测试迁移单个张量
|
||||
"""
|
||||
|
||||
paddle_tensor = paddle.rand((3, 4, 5)).cpu()
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device=None)
|
||||
self.check_cpu(res)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None)
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:1", data_device=None)
|
||||
self.check_gpu(res, 1)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device="cuda:0", data_device="cpu")
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0")
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="cuda:1")
|
||||
self.check_gpu(res, 1)
|
||||
|
||||
torch_tensor = torch.rand(3, 4, 5)
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device=None)
|
||||
self.check_cpu(res)
|
||||
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device=None)
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:1", data_device=None)
|
||||
self.check_gpu(res, 1)
|
||||
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device="cpu")
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:0")
|
||||
self.check_gpu(res, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:1")
|
||||
self.check_gpu(res, 1)
|
||||
|
||||
def test_list_transfer(self):
|
||||
"""
|
||||
测试迁移张量的列表
|
||||
"""
|
||||
|
||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)]
|
||||
res = torch_paddle_move_data_to_device(paddle_list, device=None, data_device="gpu:1")
|
||||
assert isinstance(res, list)
|
||||
for r in res:
|
||||
self.check_gpu(r, 1)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1")
|
||||
assert isinstance(res, list)
|
||||
for r in res:
|
||||
self.check_cpu(r)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_list, device="gpu:0", data_device=None)
|
||||
assert isinstance(res, list)
|
||||
for r in res:
|
||||
self.check_gpu(r, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_list, device="gpu:1", data_device="cpu")
|
||||
assert isinstance(res, list)
|
||||
for r in res:
|
||||
self.check_gpu(r, 1)
|
||||
|
||||
def test_tensor_tuple_transfer(self):
|
||||
"""
|
||||
测试迁移张量的元组
|
||||
"""
|
||||
|
||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] + [torch.rand((6, 4, 2)) for i in range(5)]
|
||||
paddle_tuple = tuple(paddle_list)
|
||||
res = torch_paddle_move_data_to_device(paddle_tuple, device=None, data_device="gpu:1")
|
||||
assert isinstance(res, tuple)
|
||||
for r in res:
|
||||
self.check_gpu(r, 1)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1")
|
||||
assert isinstance(res, tuple)
|
||||
for r in res:
|
||||
self.check_cpu(r)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:0", data_device=None)
|
||||
assert isinstance(res, tuple)
|
||||
for r in res:
|
||||
self.check_gpu(r, 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:1", data_device="cpu")
|
||||
assert isinstance(res, tuple)
|
||||
for r in res:
|
||||
self.check_gpu(r, 1)
|
||||
|
||||
def test_dict_transfer(self):
|
||||
"""
|
||||
测试迁移复杂的字典结构
|
||||
"""
|
||||
|
||||
paddle_dict = {
|
||||
"torch_tensor": torch.rand((3, 4)),
|
||||
"torch_list": [torch.rand((6, 4, 2)) for i in range(10)],
|
||||
"dict":{
|
||||
"list": [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)],
|
||||
"torch_tensor": torch.rand((3, 4)),
|
||||
"paddle_tensor": paddle.rand((3, 4))
|
||||
},
|
||||
"paddle_tensor": paddle.rand((3, 4)),
|
||||
"list": [paddle.rand((6, 4, 2)) for i in range(10)] ,
|
||||
"int": 2,
|
||||
"string": "test string"
|
||||
}
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device=None)
|
||||
assert isinstance(res, dict)
|
||||
self.check_gpu(res["torch_tensor"], 0)
|
||||
self.check_gpu(res["paddle_tensor"], 0)
|
||||
assert isinstance(res["torch_list"], list)
|
||||
for t in res["torch_list"]:
|
||||
self.check_gpu(t, 0)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_gpu(t, 0)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_gpu(t, 0)
|
||||
self.check_gpu(res["dict"]["torch_tensor"], 0)
|
||||
self.check_gpu(res["dict"]["paddle_tensor"], 0)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_dict, device=None, data_device="gpu:1")
|
||||
assert isinstance(res, dict)
|
||||
self.check_gpu(res["torch_tensor"], 1)
|
||||
self.check_gpu(res["paddle_tensor"], 1)
|
||||
assert isinstance(res["torch_list"], list)
|
||||
for t in res["torch_list"]:
|
||||
self.check_gpu(t, 1)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_gpu(t, 1)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_gpu(t, 1)
|
||||
self.check_gpu(res["dict"]["torch_tensor"], 1)
|
||||
self.check_gpu(res["dict"]["paddle_tensor"], 1)
|
||||
|
||||
res = torch_paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0")
|
||||
assert isinstance(res, dict)
|
||||
self.check_cpu(res["torch_tensor"])
|
||||
self.check_cpu(res["paddle_tensor"])
|
||||
assert isinstance(res["torch_list"], list)
|
||||
for t in res["torch_list"]:
|
||||
self.check_cpu(t)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_cpu(t)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_cpu(t)
|
||||
self.check_cpu(res["dict"]["torch_tensor"])
|
||||
self.check_cpu(res["dict"]["paddle_tensor"])
|
||||
@ -1,378 +0,0 @@
|
||||
import pytest
|
||||
import os
|
||||
from itertools import chain
|
||||
|
||||
import torch
|
||||
import paddle
|
||||
from paddle.io import Dataset, DataLoader
|
||||
import numpy as np
|
||||
|
||||
from fastNLP.modules.mix_modules.mix_module import MixModule
|
||||
from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle
|
||||
from fastNLP.envs.distributed import rank_zero_rm
|
||||
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试类的基本功能
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class MixModuleForTest(MixModule):
|
||||
def __init__(self):
|
||||
super(MixModuleForTest, self).__init__()
|
||||
|
||||
self.torch_fc1 = torch.nn.Linear(10, 10)
|
||||
self.torch_softmax = torch.nn.Softmax(0)
|
||||
self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3)
|
||||
self.torch_tensor = torch.ones(3, 3)
|
||||
self.torch_param = torch.nn.Parameter(torch.ones(4, 4))
|
||||
|
||||
self.paddle_fc1 = paddle.nn.Linear(10, 10)
|
||||
self.paddle_softmax = paddle.nn.Softmax(0)
|
||||
self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3)
|
||||
self.paddle_tensor = paddle.ones((4, 4))
|
||||
|
||||
class TorchModuleForTest(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super(TorchModuleForTest, self).__init__()
|
||||
|
||||
self.torch_fc1 = torch.nn.Linear(10, 10)
|
||||
self.torch_softmax = torch.nn.Softmax(0)
|
||||
self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3)
|
||||
self.torch_tensor = torch.ones(3, 3)
|
||||
self.torch_param = torch.nn.Parameter(torch.ones(4, 4))
|
||||
|
||||
class PaddleModuleForTest(paddle.nn.Layer):
|
||||
def __init__(self):
|
||||
super(PaddleModuleForTest, self).__init__()
|
||||
|
||||
self.paddle_fc1 = paddle.nn.Linear(10, 10)
|
||||
self.paddle_softmax = paddle.nn.Softmax(0)
|
||||
self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3)
|
||||
self.paddle_tensor = paddle.ones((4, 4))
|
||||
|
||||
|
||||
@pytest.mark.torchpaddle
|
||||
class TestTorchPaddleMixModule:
|
||||
|
||||
def setup_method(self):
|
||||
|
||||
self.model = MixModuleForTest()
|
||||
self.torch_model = TorchModuleForTest()
|
||||
self.paddle_model = PaddleModuleForTest()
|
||||
|
||||
def test_to(self):
|
||||
"""
|
||||
测试混合模型的to函数
|
||||
"""
|
||||
|
||||
self.model.to("cuda")
|
||||
self.torch_model.to("cuda")
|
||||
self.paddle_model.to("gpu")
|
||||
self.if_device_correct("cuda")
|
||||
|
||||
self.model.to("cuda:2")
|
||||
self.torch_model.to("cuda:2")
|
||||
self.paddle_model.to("gpu:2")
|
||||
self.if_device_correct("cuda:2")
|
||||
|
||||
self.model.to("gpu:1")
|
||||
self.torch_model.to("cuda:1")
|
||||
self.paddle_model.to("gpu:1")
|
||||
self.if_device_correct("cuda:1")
|
||||
|
||||
self.model.to("cpu")
|
||||
self.torch_model.to("cpu")
|
||||
self.paddle_model.to("cpu")
|
||||
self.if_device_correct("cpu")
|
||||
|
||||
def test_train_eval(self):
|
||||
"""
|
||||
测试train和eval函数
|
||||
"""
|
||||
|
||||
self.model.eval()
|
||||
self.if_training_correct(False)
|
||||
|
||||
self.model.train()
|
||||
self.if_training_correct(True)
|
||||
|
||||
def test_parameters(self):
|
||||
"""
|
||||
测试parameters()函数，由于初始化是随机的，目前仅比较得到结果的长度
|
||||
"""
|
||||
mix_params = []
|
||||
params = []
|
||||
|
||||
for value in self.model.named_parameters():
|
||||
mix_params.append(value)
|
||||
|
||||
for value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()):
|
||||
params.append(value)
|
||||
|
||||
assert len(params) == len(mix_params)
|
||||
|
||||
def test_named_parameters(self):
|
||||
"""
|
||||
测试named_parameters函数
|
||||
"""
|
||||
|
||||
mix_param_names = []
|
||||
param_names = []
|
||||
|
||||
for name, value in self.model.named_parameters():
|
||||
mix_param_names.append(name)
|
||||
|
||||
for name, value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()):
|
||||
param_names.append(name)
|
||||
|
||||
assert sorted(param_names) == sorted(mix_param_names)
|
||||
|
||||
def test_torch_named_parameters(self):
|
||||
"""
|
||||
测试对torch参数的提取
|
||||
"""
|
||||
|
||||
mix_param_names = []
|
||||
param_names = []
|
||||
|
||||
for name, value in self.model.named_parameters(backend="torch"):
|
||||
mix_param_names.append(name)
|
||||
|
||||
for name, value in self.torch_model.named_parameters():
|
||||
param_names.append(name)
|
||||
|
||||
assert sorted(param_names) == sorted(mix_param_names)
|
||||
|
||||
def test_paddle_named_parameters(self):
|
||||
"""
|
||||
测试对paddle参数的提取
|
||||
"""
|
||||
|
||||
mix_param_names = []
|
||||
param_names = []
|
||||
|
||||
for name, value in self.model.named_parameters(backend="paddle"):
|
||||
mix_param_names.append(name)
|
||||
|
||||
for name, value in self.paddle_model.named_parameters():
|
||||
param_names.append(name)
|
||||
|
||||
assert sorted(param_names) == sorted(mix_param_names)
|
||||
|
||||
def test_torch_state_dict(self):
|
||||
"""
|
||||
测试提取torch的state dict
|
||||
"""
|
||||
torch_dict = self.torch_model.state_dict()
|
||||
mix_dict = self.model.state_dict(backend="torch")
|
||||
|
||||
assert sorted(torch_dict.keys()) == sorted(mix_dict.keys())
|
||||
|
||||
def test_paddle_state_dict(self):
|
||||
"""
|
||||
测试提取paddle的state dict
|
||||
"""
|
||||
paddle_dict = self.paddle_model.state_dict()
|
||||
mix_dict = self.model.state_dict(backend="paddle")
|
||||
|
||||
# TODO 测试程序会显示passed后显示paddle的异常退出信息
|
||||
assert sorted(paddle_dict.keys()) == sorted(mix_dict.keys())
|
||||
|
||||
def test_state_dict(self):
|
||||
"""
|
||||
测试提取所有的state dict
|
||||
"""
|
||||
all_dict = self.torch_model.state_dict()
|
||||
all_dict.update(self.paddle_model.state_dict())
|
||||
mix_dict = self.model.state_dict()
|
||||
|
||||
# TODO 测试程序会显示passed后显示paddle的异常退出信息
|
||||
assert sorted(all_dict.keys()) == sorted(mix_dict.keys())
|
||||
|
||||
def test_load_state_dict(self):
|
||||
"""
|
||||
测试load_state_dict函数
|
||||
"""
|
||||
state_dict = self.model.state_dict()
|
||||
|
||||
new_model = MixModuleForTest()
|
||||
new_model.load_state_dict(state_dict)
|
||||
new_state_dict = new_model.state_dict()
|
||||
|
||||
for name, value in state_dict.items():
|
||||
state_dict[name] = value.tolist()
|
||||
for name, value in new_state_dict.items():
|
||||
new_state_dict[name] = value.tolist()
|
||||
|
||||
# self.assertDictEqual(state_dict, new_state_dict)
|
||||
|
||||
def test_save_and_load_state_dict(self):
|
||||
"""
|
||||
测试save_state_dict_to_file和load_state_dict_from_file函数
|
||||
"""
|
||||
path = "model"
|
||||
try:
|
||||
self.model.save_state_dict_to_file(path)
|
||||
new_model = MixModuleForTest()
|
||||
new_model.load_state_dict_from_file(path)
|
||||
|
||||
state_dict = self.model.state_dict()
|
||||
new_state_dict = new_model.state_dict()
|
||||
|
||||
for name, value in state_dict.items():
|
||||
state_dict[name] = value.tolist()
|
||||
for name, value in new_state_dict.items():
|
||||
new_state_dict[name] = value.tolist()
|
||||
|
||||
# self.assertDictEqual(state_dict, new_state_dict)
|
||||
finally:
|
||||
rank_zero_rm(path)
|
||||
|
||||
def if_device_correct(self, device):
|
||||
|
||||
|
||||
assert self.model.torch_fc1.weight.device == self.torch_model.torch_fc1.weight.device
|
||||
assert self.model.torch_conv2d1.weight.device == self.torch_model.torch_fc1.bias.device
|
||||
assert self.model.torch_conv2d1.bias.device == self.torch_model.torch_conv2d1.bias.device
|
||||
assert self.model.torch_tensor.device == self.torch_model.torch_tensor.device
|
||||
assert self.model.torch_param.device == self.torch_model.torch_param.device
|
||||
|
||||
if device == "cpu":
|
||||
assert self.model.paddle_fc1.weight.place.is_cpu_place()
|
||||
assert self.model.paddle_fc1.bias.place.is_cpu_place()
|
||||
assert self.model.paddle_conv2d1.weight.place.is_cpu_place()
|
||||
assert self.model.paddle_conv2d1.bias.place.is_cpu_place()
|
||||
assert self.model.paddle_tensor.place.is_cpu_place()
|
||||
elif device.startswith("cuda"):
|
||||
assert self.model.paddle_fc1.weight.place.is_gpu_place()
|
||||
assert self.model.paddle_fc1.bias.place.is_gpu_place()
|
||||
assert self.model.paddle_conv2d1.weight.place.is_gpu_place()
|
||||
assert self.model.paddle_conv2d1.bias.place.is_gpu_place()
|
||||
assert self.model.paddle_tensor.place.is_gpu_place()
|
||||
|
||||
assert self.model.paddle_fc1.weight.place.gpu_device_id() == self.paddle_model.paddle_fc1.weight.place.gpu_device_id()
|
||||
assert self.model.paddle_fc1.bias.place.gpu_device_id() == self.paddle_model.paddle_fc1.bias.place.gpu_device_id()
|
||||
assert self.model.paddle_conv2d1.weight.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.weight.place.gpu_device_id()
|
||||
assert self.model.paddle_conv2d1.bias.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.bias.place.gpu_device_id()
|
||||
assert self.model.paddle_tensor.place.gpu_device_id() == self.paddle_model.paddle_tensor.place.gpu_device_id()
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
def if_training_correct(self, training):
|
||||
|
||||
assert self.model.torch_fc1.training == training
|
||||
assert self.model.torch_softmax.training == training
|
||||
assert self.model.torch_conv2d1.training == training
|
||||
|
||||
assert self.model.paddle_fc1.training == training
|
||||
assert self.model.paddle_softmax.training == training
|
||||
assert self.model.paddle_conv2d1.training == training
|
||||
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试在MNIST数据集上的表现
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class MNISTDataset(Dataset):
|
||||
def __init__(self, dataset):
|
||||
|
||||
self.dataset = [
|
||||
(
|
||||
np.array(img).astype('float32').reshape(-1),
|
||||
label
|
||||
) for img, label in dataset
|
||||
]
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.dataset[idx]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
class MixMNISTModel(MixModule):
|
||||
def __init__(self):
|
||||
super(MixMNISTModel, self).__init__()
|
||||
|
||||
self.fc1 = paddle.nn.Linear(784, 64)
|
||||
self.fc2 = paddle.nn.Linear(64, 32)
|
||||
self.fc3 = torch.nn.Linear(32, 10)
|
||||
self.fc4 = torch.nn.Linear(10, 10)
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
paddle_out = self.fc1(x)
|
||||
paddle_out = self.fc2(paddle_out)
|
||||
torch_in = paddle2torch(paddle_out)
|
||||
torch_out = self.fc3(torch_in)
|
||||
torch_out = self.fc4(torch_out)
|
||||
|
||||
return torch_out
|
||||
|
||||
@pytest.mark.torchpaddle
|
||||
class TestMNIST:
|
||||
|
||||
@classmethod
|
||||
def setup_class(self):
|
||||
|
||||
self.train_dataset = paddle.vision.datasets.MNIST(mode='train')
|
||||
self.test_dataset = paddle.vision.datasets.MNIST(mode='test')
|
||||
self.train_dataset = MNISTDataset(self.train_dataset)
|
||||
|
||||
self.lr = 0.0003
|
||||
self.epochs = 20
|
||||
|
||||
self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True)
|
||||
|
||||
def setup_method(self):
|
||||
|
||||
self.model = MixMNISTModel().to("cuda")
|
||||
self.torch_loss_func = torch.nn.CrossEntropyLoss()
|
||||
|
||||
self.torch_opt = torch.optim.Adam(self.model.parameters(backend="torch"), self.lr)
|
||||
self.paddle_opt = paddle.optimizer.Adam(parameters=self.model.parameters(backend="paddle"), learning_rate=self.lr)
|
||||
|
||||
def test_case1(self):
|
||||
|
||||
# 开始训练
|
||||
for epoch in range(self.epochs):
|
||||
epoch_loss, batch = 0, 0
|
||||
for batch, (img, label) in enumerate(self.dataloader):
|
||||
|
||||
img = paddle.to_tensor(img).cuda()
|
||||
torch_out = self.model(img)
|
||||
label = torch.from_numpy(label.numpy()).reshape(-1)
|
||||
loss = self.torch_loss_func(torch_out.cpu(), label)
|
||||
epoch_loss += loss.item()
|
||||
|
||||
loss.backward()
|
||||
self.torch_opt.step()
|
||||
self.paddle_opt.step()
|
||||
self.torch_opt.zero_grad()
|
||||
self.paddle_opt.clear_grad()
|
||||
|
||||
else:
|
||||
assert epoch_loss / (batch + 1) < 0.3
|
||||
|
||||
# 开始测试
|
||||
correct = 0
|
||||
for img, label in self.test_dataset:
|
||||
|
||||
img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1))
|
||||
torch_out = self.model(img)
|
||||
res = torch_out.softmax(-1).argmax().item()
|
||||
label = label.item()
|
||||
if res == label:
|
||||
correct += 1
|
||||
|
||||
acc = correct / len(self.test_dataset)
|
||||
assert acc > 0.85
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试在ERNIE中文数据集上的表现
|
||||
#
|
||||
############################################################################
|
||||
@ -1,435 +0,0 @@
|
||||
import unittest
|
||||
import os
|
||||
|
||||
os.environ["log_silent"] = "1"
|
||||
import torch
|
||||
import paddle
|
||||
import jittor
|
||||
|
||||
from fastNLP.modules.mix_modules.utils import (
|
||||
paddle2torch,
|
||||
torch2paddle,
|
||||
jittor2torch,
|
||||
torch2jittor,
|
||||
)
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试paddle到torch的转换
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class Paddle2TorchTestCase(unittest.TestCase):
|
||||
|
||||
def check_torch_tensor(self, tensor, device, requires_grad):
|
||||
"""
|
||||
检查张量设备和梯度情况的工具函数
|
||||
"""
|
||||
|
||||
assert isinstance(tensor, torch.Tensor)
|
||||
assert tensor.device == torch.device(device)
|
||||
assert tensor.requires_grad == requires_grad
|
||||
|
||||
def test_gradient(self):
|
||||
"""
|
||||
测试张量转换后的反向传播是否正确
|
||||
"""
|
||||
|
||||
x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0], stop_gradient=False)
|
||||
y = paddle2torch(x)
|
||||
z = 3 * (y ** 2)
|
||||
z.sum().backward()
|
||||
assert y.grad.tolist() == [6, 12, 18, 24, 30]
|
||||
|
||||
def test_tensor_transfer(self):
|
||||
"""
|
||||
测试单个张量的设备和梯度转换是否正确
|
||||
"""
|
||||
|
||||
paddle_tensor = paddle.rand((3, 4, 5)).cpu()
|
||||
res = paddle2torch(paddle_tensor)
|
||||
self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient)
|
||||
|
||||
res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None)
|
||||
self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient)
|
||||
|
||||
res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True)
|
||||
self.check_torch_tensor(res, "cuda:1", False)
|
||||
|
||||
res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False)
|
||||
self.check_torch_tensor(res, "cuda:1", True)
|
||||
|
||||
def test_list_transfer(self):
|
||||
"""
|
||||
测试张量列表的转换
|
||||
"""
|
||||
|
||||
paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)]
|
||||
res = paddle2torch(paddle_list)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cuda:1", False)
|
||||
|
||||
res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cpu", True)
|
||||
|
||||
def test_tensor_tuple_transfer(self):
|
||||
"""
|
||||
测试张量元组的转换
|
||||
"""
|
||||
|
||||
paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)]
|
||||
paddle_tuple = tuple(paddle_list)
|
||||
res = paddle2torch(paddle_tuple)
|
||||
assert isinstance(res, tuple)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cuda:1", False)
|
||||
|
||||
def test_dict_transfer(self):
|
||||
"""
|
||||
测试包含复杂结构的字典的转换
|
||||
"""
|
||||
|
||||
paddle_dict = {
|
||||
"tensor": paddle.rand((3, 4)).cuda(0),
|
||||
"list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)],
|
||||
"dict":{
|
||||
"list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)],
|
||||
"tensor": paddle.rand((3, 4)).cuda(0)
|
||||
},
|
||||
"int": 2,
|
||||
"string": "test string"
|
||||
}
|
||||
res = paddle2torch(paddle_dict)
|
||||
assert isinstance(res, dict)
|
||||
self.check_torch_tensor(res["tensor"], "cuda:0", False)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_torch_tensor(t, "cuda:0", False)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_torch_tensor(t, "cuda:0", False)
|
||||
self.check_torch_tensor(res["dict"]["tensor"], "cuda:0", False)
|
||||
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试torch到paddle的转换
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class Torch2PaddleTestCase(unittest.TestCase):
|
||||
|
||||
def check_paddle_tensor(self, tensor, device, stop_gradient):
|
||||
"""
|
||||
检查得到的paddle张量设备和梯度情况的工具函数
|
||||
"""
|
||||
|
||||
assert isinstance(tensor, paddle.Tensor)
|
||||
if device == "cpu":
|
||||
assert tensor.place.is_cpu_place()
|
||||
elif device.startswith("gpu"):
|
||||
paddle_device = paddle.device._convert_to_place(device)
|
||||
assert tensor.place.is_gpu_place()
|
||||
if hasattr(tensor.place, "gpu_device_id"):
|
||||
# paddle中，有两种Place
|
||||
# paddle.fluid.core.Place是创建Tensor时使用的类型
|
||||
# 有函数gpu_device_id获取设备
|
||||
assert tensor.place.gpu_device_id() == paddle_device.get_device_id()
|
||||
else:
|
||||
# 通过_convert_to_place得到的是paddle.CUDAPlace
|
||||
# 通过get_device_id获取设备
|
||||
assert tensor.place.get_device_id() == paddle_device.get_device_id()
|
||||
else:
|
||||
raise NotImplementedError
|
||||
assert tensor.stop_gradient == stop_gradient
|
||||
|
||||
def test_gradient(self):
|
||||
"""
|
||||
测试转换后梯度的反向传播
|
||||
"""
|
||||
|
||||
x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True)
|
||||
y = torch2paddle(x)
|
||||
z = 3 * (y ** 2)
|
||||
z.sum().backward()
|
||||
assert y.grad.tolist() == [6, 12, 18, 24, 30]
|
||||
|
||||
def test_tensor_transfer(self):
|
||||
"""
|
||||
测试单个张量的转换
|
||||
"""
|
||||
|
||||
torch_tensor = torch.rand((3, 4, 5))
|
||||
res = torch2paddle(torch_tensor)
|
||||
self.check_paddle_tensor(res, "cpu", True)
|
||||
|
||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None)
|
||||
self.check_paddle_tensor(res, "gpu:2", True)
|
||||
|
||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True)
|
||||
self.check_paddle_tensor(res, "gpu:2", True)
|
||||
|
||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False)
|
||||
self.check_paddle_tensor(res, "gpu:2", False)
|
||||
|
||||
def test_tensor_list_transfer(self):
|
||||
"""
|
||||
测试张量列表的转换
|
||||
"""
|
||||
|
||||
torch_list = [torch.rand(6, 4, 2) for i in range(10)]
|
||||
res = torch2paddle(torch_list)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_paddle_tensor(t, "cpu", True)
|
||||
|
||||
res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_paddle_tensor(t, "gpu:1", False)
|
||||
|
||||
def test_tensor_tuple_transfer(self):
|
||||
"""
|
||||
测试张量元组的转换
|
||||
"""
|
||||
|
||||
torch_list = [torch.rand(6, 4, 2) for i in range(10)]
|
||||
torch_tuple = tuple(torch_list)
|
||||
res = torch2paddle(torch_tuple, target_device="cpu")
|
||||
assert isinstance(res, tuple)
|
||||
for t in res:
|
||||
self.check_paddle_tensor(t, "cpu", True)
|
||||
|
||||
def test_dict_transfer(self):
|
||||
"""
|
||||
测试复杂的字典结构的转换
|
||||
"""
|
||||
|
||||
torch_dict = {
|
||||
"tensor": torch.rand((3, 4)),
|
||||
"list": [torch.rand(6, 4, 2) for i in range(10)],
|
||||
"dict":{
|
||||
"list": [torch.rand(6, 4, 2) for i in range(10)],
|
||||
"tensor": torch.rand((3, 4))
|
||||
},
|
||||
"int": 2,
|
||||
"string": "test string"
|
||||
}
|
||||
res = torch2paddle(torch_dict)
|
||||
assert isinstance(res, dict)
|
||||
self.check_paddle_tensor(res["tensor"], "cpu", True)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_paddle_tensor(t, "cpu", True)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_paddle_tensor(t, "cpu", True)
|
||||
self.check_paddle_tensor(res["dict"]["tensor"], "cpu", True)
|
||||
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试jittor到torch的转换
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class Jittor2TorchTestCase(unittest.TestCase):
|
||||
|
||||
def check_torch_tensor(self, tensor, device, requires_grad):
|
||||
"""
|
||||
检查得到的torch张量的工具函数
|
||||
"""
|
||||
|
||||
assert isinstance(tensor, torch.Tensor)
|
||||
if device == "cpu":
|
||||
assert not tensor.is_cuda
|
||||
else:
|
||||
assert tensor.device == torch.device(device)
|
||||
assert tensor.requires_grad == requires_grad
|
||||
|
||||
def test_var_transfer(self):
|
||||
"""
|
||||
测试单个Jittor Var的转换
|
||||
"""
|
||||
|
||||
jittor_var = jittor.rand((3, 4, 5))
|
||||
res = jittor2torch(jittor_var)
|
||||
self.check_torch_tensor(res, "cpu", True)
|
||||
|
||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None)
|
||||
self.check_torch_tensor(res, "cuda:2", True)
|
||||
|
||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True)
|
||||
self.check_torch_tensor(res, "cuda:2", False)
|
||||
|
||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False)
|
||||
self.check_torch_tensor(res, "cuda:2", True)
|
||||
|
||||
def test_var_list_transfer(self):
|
||||
"""
|
||||
测试Jittor列表的转换
|
||||
"""
|
||||
|
||||
jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)]
|
||||
res = jittor2torch(jittor_list)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cpu", True)
|
||||
|
||||
res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cuda:1", True)
|
||||
|
||||
def test_var_tuple_transfer(self):
|
||||
"""
|
||||
测试Jittor变量元组的转换
|
||||
"""
|
||||
|
||||
jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)]
|
||||
jittor_tuple = tuple(jittor_list)
|
||||
res = jittor2torch(jittor_tuple, target_device="cpu")
|
||||
assert isinstance(res, tuple)
|
||||
for t in res:
|
||||
self.check_torch_tensor(t, "cpu", True)
|
||||
|
||||
def test_dict_transfer(self):
|
||||
"""
|
||||
测试字典结构的转换
|
||||
"""
|
||||
|
||||
jittor_dict = {
|
||||
"tensor": jittor.rand((3, 4)),
|
||||
"list": [jittor.rand(6, 4, 2) for i in range(10)],
|
||||
"dict":{
|
||||
"list": [jittor.rand(6, 4, 2) for i in range(10)],
|
||||
"tensor": jittor.rand((3, 4))
|
||||
},
|
||||
"int": 2,
|
||||
"string": "test string"
|
||||
}
|
||||
res = jittor2torch(jittor_dict)
|
||||
assert isinstance(res, dict)
|
||||
self.check_torch_tensor(res["tensor"], "cpu", True)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_torch_tensor(t, "cpu", True)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_torch_tensor(t, "cpu", True)
|
||||
self.check_torch_tensor(res["dict"]["tensor"], "cpu", True)
|
||||
|
||||
|
||||
############################################################################
|
||||
#
|
||||
# 测试torch到jittor的转换
|
||||
#
|
||||
############################################################################
|
||||
|
||||
class Torch2JittorTestCase(unittest.TestCase):
|
||||
|
||||
def check_jittor_var(self, var, requires_grad):
|
||||
"""
|
||||
检查得到的Jittor Var梯度情况的工具函数
|
||||
"""
|
||||
|
||||
assert isinstance(var, jittor.Var)
|
||||
assert var.requires_grad == requires_grad
|
||||
|
||||
def test_gradient(self):
|
||||
"""
|
||||
测试反向传播的梯度
|
||||
"""
|
||||
|
||||
x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True)
|
||||
y = torch2jittor(x)
|
||||
z = 3 * (y ** 2)
|
||||
grad = jittor.grad(z, y)
|
||||
assert grad.tolist() == [6.0, 12.0, 18.0, 24.0, 30.0]
|
||||
|
||||
def test_tensor_transfer(self):
|
||||
"""
|
||||
测试单个张量转换为Jittor
|
||||
"""
|
||||
|
||||
torch_tensor = torch.rand((3, 4, 5))
|
||||
res = torch2jittor(torch_tensor)
|
||||
self.check_jittor_var(res, False)
|
||||
|
||||
res = torch2jittor(torch_tensor, no_gradient=None)
|
||||
self.check_jittor_var(res, False)
|
||||
|
||||
res = torch2jittor(torch_tensor, no_gradient=True)
|
||||
self.check_jittor_var(res, False)
|
||||
|
||||
res = torch2jittor(torch_tensor, no_gradient=False)
|
||||
self.check_jittor_var(res, True)
|
||||
|
||||
def test_tensor_list_transfer(self):
|
||||
"""
|
||||
测试张量列表的转换
|
||||
"""
|
||||
|
||||
torch_list = [torch.rand((6, 4, 2)) for i in range(10)]
|
||||
res = torch2jittor(torch_list)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_jittor_var(t, False)
|
||||
|
||||
res = torch2jittor(torch_list, no_gradient=False)
|
||||
assert isinstance(res, list)
|
||||
for t in res:
|
||||
self.check_jittor_var(t, True)
|
||||
|
||||
def test_tensor_tuple_transfer(self):
|
||||
"""
|
||||
测试张量元组的转换
|
||||
"""
|
||||
|
||||
torch_list = [torch.rand((6, 4, 2)) for i in range(10)]
|
||||
torch_tuple = tuple(torch_list)
|
||||
res = torch2jittor(torch_tuple)
|
||||
assert isinstance(res, tuple)
|
||||
for t in res:
|
||||
self.check_jittor_var(t, False)
|
||||
|
||||
def test_dict_transfer(self):
|
||||
"""
|
||||
测试字典结构的转换
|
||||
"""
|
||||
|
||||
torch_dict = {
|
||||
"tensor": torch.rand((3, 4)),
|
||||
"list": [torch.rand(6, 4, 2) for i in range(10)],
|
||||
"dict":{
|
||||
"list": [torch.rand(6, 4, 2) for i in range(10)],
|
||||
"tensor": torch.rand((3, 4))
|
||||
},
|
||||
"int": 2,
|
||||
"string": "test string"
|
||||
}
|
||||
res = torch2jittor(torch_dict)
|
||||
assert isinstance(res, dict)
|
||||
self.check_jittor_var(res["tensor"], False)
|
||||
assert isinstance(res["list"], list)
|
||||
for t in res["list"]:
|
||||
self.check_jittor_var(t, False)
|
||||
assert isinstance(res["int"], int)
|
||||
assert isinstance(res["string"], str)
|
||||
assert isinstance(res["dict"], dict)
|
||||
assert isinstance(res["dict"]["list"], list)
|
||||
for t in res["dict"]["list"]:
|
||||
self.check_jittor_var(t, False)
|
||||
self.check_jittor_var(res["dict"]["tensor"], False)
|
||||
Loading…
Reference in New Issue
Block a user