sst loader中增加一列raw_words

2024-12-02 20:27:35 +08:00 · 2019-07-25 16:56:54 +08:00 · 2019-07-25 16:56:54 +08:00 · d401bd2208
commit d401bd2208
parent cb3cb8bc5c
1 changed files with 1 additions and 0 deletions
--- a/fastNLP/io/data_loader/sst.py
+++ b/fastNLP/io/data_loader/sst.py
@ -134,6 +134,7 @@ class SST2Loader(CSVLoader):
        info = DataBundle()
        for name, path in paths.items():
            dataset = self.load(path)
+            dataset.apply_field(lambda words:words.copy(), field_name='words', new_field_name='raw_words')
            datasets[name] = dataset

        def wordtochar(words):