test/dify
1
0
Fork 0
mirror of https://gitee.com/dify_ai/dify.git synced 2024-12-02 03:07:59 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

ascii filter Unicode U+FFFE (#2038)

Browse Source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
Jyong 2024-01-15 16:52:18 +08:00 committed by GitHub
parent bdb0d77227
commit cb7a608d75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/core/indexing_runner.py
View File

@ -531,7 +531,9 @@ class IndexingRunner:
def filter_string(self, text): def filter_string(self, text):
text = re.sub(r'<\|', '<', text) text = re.sub(r'<\|', '<', text)
text = re.sub(r'\|>', '>', text) text = re.sub(r'\|>', '>', text)
text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]', '', text) text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]', '', text)
# Unicode U+FFFE
text = re.sub(u'\uFFFE', '', text)
return text return text
def _get_splitter(self, processing_rule: DatasetProcessRule, def _get_splitter(self, processing_rule: DatasetProcessRule,