test/dify
1
0
Fork 0
mirror of https://gitee.com/dify_ai/dify.git synced 2024-11-30 02:08:37 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Fix/json format (#466)

Browse Source
This commit is contained in:
Jyong 2023-06-28 13:58:50 +08:00 committed by GitHub
parent 2eea114ac0
commit 334f46d0b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/core/data_loader/loader/excel.py
View File

@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader):
row_dict = dict(zip(keys, list(map(str, row))))
row_dict = {k: v for k, v in row_dict.items() if v}
item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items())
data.append(item)
document = Document(page_content=item)
data.append(document)
return [Document(page_content='\n\n'.join(data))]
return data

14
api/core/data_loader/loader/notion.py
View File

@ -81,8 +81,8 @@ class NotionLoader(BaseLoader):
docs = []
if notion_page_type == 'database':
# get all the pages in the database
page_text = self._get_notion_database_data(notion_obj_id)
docs.append(Document(page_content=page_text))
page_text_documents = self._get_notion_database_data(notion_obj_id)
docs.extend(page_text_documents)
elif notion_page_type == 'page':
page_text_list = self._get_notion_block_data(notion_obj_id)
for page_text in page_text_list:
@ -94,7 +94,7 @@ class NotionLoader(BaseLoader):
def _get_notion_database_data(
self, database_id: str, query_dict: Dict[str, Any] = {}
) -> str:
) -> List[Document]:
"""Get all the pages from a Notion database."""
res = requests.post(
DATABASE_URL_TMPL.format(database_id=database_id),
@ -110,7 +110,7 @@ class NotionLoader(BaseLoader):
database_content_list = []
if 'results' not in data or data["results"] is None:
return ""
return []
for result in data["results"]:
properties = result['properties']
data = {}
@ -143,10 +143,10 @@ class NotionLoader(BaseLoader):
row_content = row_content + f'{key}:{value_content}\n'
else:
row_content = row_content + f'{key}:{value}\n'
database_content_list.append(row_content)
database_content_list.append(json.dumps(data, ensure_ascii=False))
document = Document(page_content=row_content)
database_content_list.append(document)
return "\n\n".join(database_content_list)
return database_content_list
def _get_notion_block_data(self, page_id: str) -> List[str]:
result_lines_arr = []