mirror of
https://gitee.com/dify_ai/dify.git
synced 2024-12-05 12:48:32 +08:00
d069c668f8
Co-authored-by: StyleZhang <jasonapring2015@outlook.com> Co-authored-by: Garfield Dai <dai.hai@foxmail.com> Co-authored-by: chenhe <guchenhe@gmail.com> Co-authored-by: jyong <jyong@dify.ai> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Yeuoly <admin@srmxy.cn>
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
import logging
|
|
from typing import Tuple
|
|
|
|
from core.entities.application_entities import AppOrchestrationConfigEntity
|
|
from core.moderation.base import ModerationAction, ModerationException
|
|
from core.moderation.factory import ModerationFactory
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ModerationFeature:
|
|
def check(self, app_id: str,
|
|
tenant_id: str,
|
|
app_orchestration_config_entity: AppOrchestrationConfigEntity,
|
|
inputs: dict,
|
|
query: str) -> Tuple[bool, dict, str]:
|
|
"""
|
|
Process sensitive_word_avoidance.
|
|
:param app_id: app id
|
|
:param tenant_id: tenant id
|
|
:param app_orchestration_config_entity: app orchestration config entity
|
|
:param inputs: inputs
|
|
:param query: query
|
|
:return:
|
|
"""
|
|
if not app_orchestration_config_entity.sensitive_word_avoidance:
|
|
return False, inputs, query
|
|
|
|
sensitive_word_avoidance_config = app_orchestration_config_entity.sensitive_word_avoidance
|
|
moderation_type = sensitive_word_avoidance_config.type
|
|
|
|
moderation_factory = ModerationFactory(
|
|
name=moderation_type,
|
|
app_id=app_id,
|
|
tenant_id=tenant_id,
|
|
config=sensitive_word_avoidance_config.config
|
|
)
|
|
|
|
moderation_result = moderation_factory.moderation_for_inputs(inputs, query)
|
|
|
|
if not moderation_result.flagged:
|
|
return False, inputs, query
|
|
|
|
if moderation_result.action == ModerationAction.DIRECT_OUTPUT:
|
|
raise ModerationException(moderation_result.preset_response)
|
|
elif moderation_result.action == ModerationAction.OVERRIDED:
|
|
inputs = moderation_result.inputs
|
|
query = moderation_result.query
|
|
|
|
return True, inputs, query
|