95 lines
3.0 KiB
Python
95 lines
3.0 KiB
Python
"""Prompt moderation orchestration.
|
||
|
||
T-604 只做本地关键词 prompt 审核：在图片下载、定价、预扣点和上游调用之前执行。
|
||
MVP 不做输出审核、不做图片审核，也不保存用户 prompt 原文。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from dataclasses import dataclass, field
|
||
|
||
from . import policy
|
||
from .providers import ModerationError, ModerationResult, Verdict
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ModerationOutcome:
|
||
stage: str
|
||
verdict: Verdict
|
||
blocked: bool
|
||
labels: tuple[str, ...] = ()
|
||
keywords: tuple[str, ...] = ()
|
||
matched_word_ids: tuple[int, ...] = ()
|
||
request_id: str = ""
|
||
reason: str = ""
|
||
result: ModerationResult | None = field(default=None, repr=False)
|
||
|
||
|
||
def _passed(stage: str) -> ModerationOutcome:
|
||
return ModerationOutcome(stage=stage, verdict=Verdict.PASS, blocked=False)
|
||
|
||
|
||
def _on_provider_failure(stage: str, exc: Exception) -> ModerationOutcome:
|
||
"""provider 不可用时的兜底：fail-closed → BLOCK，fail-open → PASS。"""
|
||
if policy.fail_closed():
|
||
logger.warning("moderation fail-closed at %s: %s", stage, exc.__class__.__name__)
|
||
return ModerationOutcome(
|
||
stage=stage,
|
||
verdict=Verdict.BLOCK,
|
||
blocked=True,
|
||
reason=f"fail-closed: {exc.__class__.__name__}",
|
||
)
|
||
logger.warning("moderation fail-open at %s: %s", stage, exc.__class__.__name__)
|
||
return ModerationOutcome(stage=stage, verdict=Verdict.PASS, blocked=False, reason="fail-open")
|
||
|
||
|
||
def _evaluate(stage: str, call) -> ModerationOutcome:
|
||
if not policy.moderation_enabled():
|
||
return _passed(stage)
|
||
try:
|
||
result = call()
|
||
except ModerationError as exc:
|
||
outcome = _on_provider_failure(stage, exc)
|
||
_record(outcome)
|
||
return outcome
|
||
|
||
outcome = ModerationOutcome(
|
||
stage=stage,
|
||
verdict=result.verdict,
|
||
blocked=policy.verdict_is_blocking(result.verdict),
|
||
labels=result.labels,
|
||
keywords=result.keywords,
|
||
matched_word_ids=tuple(result.raw.get("word_ids", ())) if isinstance(result.raw, dict) else (),
|
||
request_id=result.request_id,
|
||
result=result,
|
||
)
|
||
_record(outcome)
|
||
return outcome
|
||
|
||
|
||
def _record(outcome: ModerationOutcome) -> None:
|
||
logger.info(
|
||
"moderation stage=%s verdict=%s blocked=%s labels=%s word_ids=%s request_id=%s",
|
||
outcome.stage,
|
||
outcome.verdict.value,
|
||
outcome.blocked,
|
||
",".join(outcome.labels),
|
||
",".join(str(word_id) for word_id in outcome.matched_word_ids),
|
||
outcome.request_id,
|
||
)
|
||
|
||
|
||
def moderate_prompt(*, user=None, api_key=None, prompt: str) -> ModerationOutcome:
|
||
return _evaluate(
|
||
"prompt",
|
||
lambda: policy.get_text_provider().moderate_text(
|
||
prompt, biz_type=policy.biz_type("input")
|
||
),
|
||
)
|
||
|
||
def moderate_input(*, user=None, prompt: str, **kwargs) -> ModerationOutcome:
|
||
return moderate_prompt(user=user, prompt=prompt)
|