test/fastNLP
1
0
Fork 0
mirror of https://gitee.com/fastnlp/fastNLP.git synced 2024-12-04 05:07:44 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

修复Vocabulary在建好词表之后新加入词导致的pad index错乱的问题

Browse Source
This commit is contained in:
yh 2019-07-06 10:55:58 +08:00
parent 29ca17d324
commit a40f57ae4a
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fastNLP/core/vocabulary.py
View File

@ -136,10 +136,10 @@ class Vocabulary(object):
"""
if self.word2idx is None:
self.word2idx = {}
if self.padding is not None:
self.word2idx[self.padding] = len(self.word2idx)
if self.unknown is not None:
self.word2idx[self.unknown] = len(self.word2idx)
if self.padding is not None:
self.word2idx[self.padding] = len(self.word2idx)
if self.unknown is not None:
self.word2idx[self.unknown] = len(self.word2idx)
max_size = min(self.max_size, len(self.word_count)) if self.max_size else None
words = self.word_count.most_common(max_size)