55 lines
1.2 KiB
Python
55 lines
1.2 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import unicodedata
|
||
|
|
from functools import lru_cache
|
||
|
|
|
||
|
|
_ZERO_WIDTH_CHARS = {
|
||
|
|
"\u200b",
|
||
|
|
"\u200c",
|
||
|
|
"\u200d",
|
||
|
|
"\ufeff",
|
||
|
|
"\u2060",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@lru_cache(maxsize=1)
|
||
|
|
def _opencc_converter():
|
||
|
|
try:
|
||
|
|
from opencc import OpenCC
|
||
|
|
except Exception:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
return OpenCC("t2s")
|
||
|
|
except Exception:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _to_simplified(value: str) -> str:
|
||
|
|
converter = _opencc_converter()
|
||
|
|
if converter is None:
|
||
|
|
return value
|
||
|
|
return converter.convert(value)
|
||
|
|
|
||
|
|
|
||
|
|
def normalize_text(value: str | None) -> str:
|
||
|
|
"""Normalize text before keyword matching.
|
||
|
|
|
||
|
|
The local keyword MVP removes obvious bypass characters only; it does not
|
||
|
|
attempt semantic rewriting or fuzzy matching.
|
||
|
|
"""
|
||
|
|
if not value:
|
||
|
|
return ""
|
||
|
|
|
||
|
|
normalized = unicodedata.normalize("NFKC", value).lower()
|
||
|
|
normalized = _to_simplified(normalized)
|
||
|
|
|
||
|
|
chars: list[str] = []
|
||
|
|
for char in normalized:
|
||
|
|
if char in _ZERO_WIDTH_CHARS:
|
||
|
|
continue
|
||
|
|
category = unicodedata.category(char)
|
||
|
|
if category[0] in {"P", "Z"}:
|
||
|
|
continue
|
||
|
|
chars.append(char)
|
||
|
|
return "".join(chars)
|