from __future__ import annotations import unicodedata from functools import lru_cache _ZERO_WIDTH_CHARS = { "\u200b", "\u200c", "\u200d", "\ufeff", "\u2060", } @lru_cache(maxsize=1) def _opencc_converter(): try: from opencc import OpenCC except Exception: return None try: return OpenCC("t2s") except Exception: return None def _to_simplified(value: str) -> str: converter = _opencc_converter() if converter is None: return value return converter.convert(value) def normalize_text(value: str | None) -> str: """Normalize text before keyword matching. The local keyword MVP removes obvious bypass characters only; it does not attempt semantic rewriting or fuzzy matching. """ if not value: return "" normalized = unicodedata.normalize("NFKC", value).lower() normalized = _to_simplified(normalized) chars: list[str] = [] for char in normalized: if char in _ZERO_WIDTH_CHARS: continue category = unicodedata.category(char) if category[0] in {"P", "Z"}: continue chars.append(char) return "".join(chars)