refactor: 将 Harness 工具移至 dev_scripts (#3)
This commit is contained in:
QiuSW
@@ -0,0 +1,394 @@
|
||||
"""Gitea Wiki 到本地 docs 镜像的共享实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote, urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CONFIG = ROOT / "wiki-docs.json"
|
||||
MIRROR_START = "<!-- gitea-wiki-mirror:start -->"
|
||||
MIRROR_END = "<!-- gitea-wiki-mirror:end -->"
|
||||
HEADER_PATTERN = re.compile(
|
||||
rf"\A{re.escape(MIRROR_START)}\n(?P<metadata>.*?)\n"
|
||||
rf"{re.escape(MIRROR_END)}\n\n(?P<body>.*)\Z",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
class WikiDocsError(RuntimeError):
|
||||
"""可供命令行直接展示的 Wiki 文档错误。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mapping:
|
||||
page: str
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
path: Path
|
||||
gitea_url: str
|
||||
owner: str
|
||||
repository: str
|
||||
mappings: tuple[Mapping, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiPage:
|
||||
title: str
|
||||
sub_url: str
|
||||
text: str
|
||||
revision: str
|
||||
html_url: str
|
||||
|
||||
|
||||
def _required_string(data: dict[str, Any], key: str) -> str:
|
||||
value = data.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise WikiDocsError(f"配置字段 {key!r} 必须是非空字符串")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def validate_mappings(raw_mappings: Any) -> tuple[Mapping, ...]:
|
||||
"""校验显式页面映射，确保只会写入 docs 下的 Markdown。"""
|
||||
|
||||
if not isinstance(raw_mappings, list) or not raw_mappings:
|
||||
raise WikiDocsError("配置字段 'mappings' 必须是非空数组")
|
||||
|
||||
mappings: list[Mapping] = []
|
||||
pages: set[str] = set()
|
||||
paths: set[str] = set()
|
||||
for index, item in enumerate(raw_mappings, start=1):
|
||||
if not isinstance(item, dict):
|
||||
raise WikiDocsError(f"第 {index} 个映射必须是对象")
|
||||
page = _required_string(item, "page")
|
||||
path = _required_string(item, "path").replace("\\", "/")
|
||||
pure_path = PurePosixPath(path)
|
||||
if (
|
||||
pure_path.is_absolute()
|
||||
or ".." in pure_path.parts
|
||||
or not pure_path.parts
|
||||
or pure_path.parts[0] != "docs"
|
||||
or pure_path.suffix.lower() != ".md"
|
||||
):
|
||||
raise WikiDocsError(f"镜像路径必须是 docs/ 下的 Markdown：{path}")
|
||||
if page in pages:
|
||||
raise WikiDocsError(f"Wiki 页面重复映射：{page}")
|
||||
if path in paths:
|
||||
raise WikiDocsError(f"本地路径重复映射：{path}")
|
||||
pages.add(page)
|
||||
paths.add(path)
|
||||
mappings.append(Mapping(page=page, path=path))
|
||||
return tuple(mappings)
|
||||
|
||||
|
||||
def load_config(path: Path = DEFAULT_CONFIG) -> Config:
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise WikiDocsError(f"无法读取 Wiki 映射配置 {path}: {exc}") from exc
|
||||
if not isinstance(raw, dict) or raw.get("schema_version") != 1:
|
||||
raise WikiDocsError("wiki-docs.json 的 schema_version 必须为 1")
|
||||
configured_url = _required_string(raw, "gitea_url")
|
||||
gitea_url = os.environ.get("GITEA_URL", configured_url).rstrip("/")
|
||||
if gitea_url.endswith("/api/v1"):
|
||||
gitea_url = gitea_url[: -len("/api/v1")]
|
||||
return Config(
|
||||
path=path,
|
||||
gitea_url=gitea_url,
|
||||
owner=_required_string(raw, "owner"),
|
||||
repository=_required_string(raw, "repository"),
|
||||
mappings=validate_mappings(raw.get("mappings")),
|
||||
)
|
||||
|
||||
|
||||
class WikiClient:
|
||||
"""只使用标准库访问 Gitea Wiki API。"""
|
||||
|
||||
def __init__(self, config: Config, token: str | None = None) -> None:
|
||||
self.config = config
|
||||
self.token = token if token is not None else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
api_path: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
query: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{self.config.gitea_url}/api/v1{api_path}"
|
||||
if query:
|
||||
url = f"{url}?{urlencode(query)}"
|
||||
headers = {"Accept": "application/json"}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
data = None
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urlopen(request, timeout=30) as response:
|
||||
body = response.read()
|
||||
except HTTPError as exc:
|
||||
if self.token and method == "GET" and exc.code in {401, 403, 404}:
|
||||
# 公共仓库可能可匿名读取，而当前 shell 中的通用令牌属于
|
||||
# 另一个实例或已失效。只对只读请求安全降级为匿名访问。
|
||||
anonymous_headers = {"Accept": "application/json"}
|
||||
anonymous_request = Request(
|
||||
url, data=data, headers=anonymous_headers, method=method
|
||||
)
|
||||
try:
|
||||
with urlopen(anonymous_request, timeout=30) as response:
|
||||
body = response.read()
|
||||
except HTTPError as anonymous_exc:
|
||||
detail = anonymous_exc.read().decode("utf-8", errors="replace")
|
||||
raise WikiDocsError(
|
||||
f"Gitea API {method} {api_path} 返回 "
|
||||
f"{anonymous_exc.code}: {detail}"
|
||||
) from anonymous_exc
|
||||
except URLError as anonymous_exc:
|
||||
raise WikiDocsError(
|
||||
f"无法连接 Gitea：{anonymous_exc.reason}"
|
||||
) from anonymous_exc
|
||||
else:
|
||||
detail = exc.read().decode("utf-8", errors="replace")
|
||||
raise WikiDocsError(
|
||||
f"Gitea API {method} {api_path} 返回 {exc.code}: {detail}"
|
||||
) from exc
|
||||
except URLError as exc:
|
||||
raise WikiDocsError(f"无法连接 Gitea：{exc.reason}") from exc
|
||||
if not body:
|
||||
return None
|
||||
try:
|
||||
return json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise WikiDocsError("Gitea API 返回了无效的 UTF-8 JSON") from exc
|
||||
|
||||
def list_pages(self) -> list[dict[str, Any]]:
|
||||
pages: list[dict[str, Any]] = []
|
||||
page_number = 1
|
||||
while True:
|
||||
batch = self._request(
|
||||
"GET",
|
||||
f"/repos/{quote(self.config.owner, safe='')}/"
|
||||
f"{quote(self.config.repository, safe='')}/wiki/pages",
|
||||
query={"page": page_number, "limit": 50},
|
||||
)
|
||||
if not isinstance(batch, list):
|
||||
raise WikiDocsError("Gitea Wiki 页面列表格式无效")
|
||||
pages.extend(item for item in batch if isinstance(item, dict))
|
||||
if len(batch) < 50:
|
||||
return pages
|
||||
page_number += 1
|
||||
|
||||
def get_page(self, page_name: str) -> WikiPage:
|
||||
metadata = next(
|
||||
(
|
||||
item
|
||||
for item in self.list_pages()
|
||||
if item.get("title") == page_name or item.get("sub_url") == page_name
|
||||
),
|
||||
None,
|
||||
)
|
||||
if metadata is None:
|
||||
raise WikiDocsError(
|
||||
f"Wiki 页面不存在：{page_name}；不会自动删除或重命名本地镜像"
|
||||
)
|
||||
sub_url = _required_string(metadata, "sub_url")
|
||||
page = self._request(
|
||||
"GET",
|
||||
f"/repos/{quote(self.config.owner, safe='')}/"
|
||||
f"{quote(self.config.repository, safe='')}/wiki/page/"
|
||||
f"{quote(sub_url, safe='%')}",
|
||||
)
|
||||
if not isinstance(page, dict):
|
||||
raise WikiDocsError(f"Wiki 页面响应格式无效：{page_name}")
|
||||
encoded_content = page.get("content_base64")
|
||||
if not isinstance(encoded_content, str):
|
||||
raise WikiDocsError(f"Wiki 页面没有 content_base64：{page_name}")
|
||||
try:
|
||||
text = base64.b64decode(encoded_content, validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise WikiDocsError(f"Wiki 页面不是有效的 UTF-8 Markdown：{page_name}") from exc
|
||||
last_commit = page.get("last_commit")
|
||||
revision = last_commit.get("sha") if isinstance(last_commit, dict) else None
|
||||
if not isinstance(revision, str) or not revision:
|
||||
raise WikiDocsError(f"Wiki 页面缺少 revision：{page_name}")
|
||||
title = page.get("title")
|
||||
resolved_title = title if isinstance(title, str) and title else page_name
|
||||
html_url = (
|
||||
f"{self.config.gitea_url}/{quote(self.config.owner, safe='')}/"
|
||||
f"{quote(self.config.repository, safe='')}/wiki/{quote(sub_url, safe='%')}"
|
||||
)
|
||||
return WikiPage(
|
||||
title=resolved_title,
|
||||
sub_url=sub_url,
|
||||
text=normalize_body(text),
|
||||
revision=revision,
|
||||
html_url=html_url,
|
||||
)
|
||||
|
||||
def create_page(self, title: str, content: str, message: str) -> WikiPage:
|
||||
if not self.token:
|
||||
raise WikiDocsError("创建 Wiki 页面需要通过 GITEA_TOKEN 提供写入令牌")
|
||||
encoded = base64.b64encode(content.encode("utf-8")).decode("ascii")
|
||||
self._request(
|
||||
"POST",
|
||||
f"/repos/{quote(self.config.owner, safe='')}/"
|
||||
f"{quote(self.config.repository, safe='')}/wiki/new",
|
||||
payload={"title": title, "content_base64": encoded, "message": message},
|
||||
)
|
||||
return self.get_page(title)
|
||||
|
||||
|
||||
def normalize_body(text: str) -> str:
|
||||
return text.replace("\r\n", "\n").replace("\r", "\n").rstrip() + "\n"
|
||||
|
||||
|
||||
def parse_mirror(text: str) -> tuple[dict[str, str], str]:
|
||||
match = HEADER_PATTERN.match(text.replace("\r\n", "\n").replace("\r", "\n"))
|
||||
if match is None:
|
||||
raise WikiDocsError("缺少或损坏 gitea-wiki-mirror 元数据头")
|
||||
metadata: dict[str, str] = {}
|
||||
for line in match.group("metadata").splitlines():
|
||||
key, separator, value = line.partition(": ")
|
||||
if not separator or not key or not value:
|
||||
raise WikiDocsError(f"无效的镜像元数据行：{line}")
|
||||
metadata[key] = value
|
||||
return metadata, normalize_body(match.group("body"))
|
||||
|
||||
|
||||
def render_mirror(page: WikiPage, existing: str | None = None) -> str:
|
||||
synchronized_at: str | None = None
|
||||
if existing is not None:
|
||||
try:
|
||||
metadata, body = parse_mirror(existing)
|
||||
except WikiDocsError:
|
||||
pass
|
||||
else:
|
||||
if metadata.get("wiki_revision") == page.revision and body == page.text:
|
||||
synchronized_at = metadata.get("synchronized_at")
|
||||
if not synchronized_at:
|
||||
synchronized_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace(
|
||||
"+00:00", "Z"
|
||||
)
|
||||
header = "\n".join(
|
||||
(
|
||||
MIRROR_START,
|
||||
"generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件)",
|
||||
f"wiki_page: {page.title}",
|
||||
f"wiki_url: {page.html_url}",
|
||||
f"wiki_revision: {page.revision}",
|
||||
f"synchronized_at: {synchronized_at}",
|
||||
MIRROR_END,
|
||||
)
|
||||
)
|
||||
return f"{header}\n\n{page.text}"
|
||||
|
||||
|
||||
def dirty_mirror_paths(config: Config, root: Path = ROOT) -> list[str]:
|
||||
paths = [mapping.path for mapping in config.mappings]
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain", "--", *paths],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return [line for line in result.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _write_atomic(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.", suffix=".tmp", dir=path.parent
|
||||
)
|
||||
try:
|
||||
with os.fdopen(handle, "w", encoding="utf-8", newline="\n") as stream:
|
||||
stream.write(content)
|
||||
os.replace(temporary_name, path)
|
||||
except BaseException:
|
||||
Path(temporary_name).unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def check_mirror(mapping: Mapping, page: WikiPage, path: Path) -> list[str]:
|
||||
if not path.is_file():
|
||||
return [f"缺少镜像：{mapping.path}"]
|
||||
try:
|
||||
metadata, body = parse_mirror(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, WikiDocsError) as exc:
|
||||
return [f"镜像无效 {mapping.path}: {exc}"]
|
||||
expected = {
|
||||
"wiki_page": page.title,
|
||||
"wiki_url": page.html_url,
|
||||
"wiki_revision": page.revision,
|
||||
}
|
||||
errors = [
|
||||
f"{mapping.path} 的 {key} 不一致"
|
||||
for key, value in expected.items()
|
||||
if metadata.get(key) != value
|
||||
]
|
||||
if not metadata.get("synchronized_at"):
|
||||
errors.append(f"{mapping.path} 缺少 synchronized_at")
|
||||
if body != page.text:
|
||||
errors.append(f"{mapping.path} 的正文与 Wiki 不一致")
|
||||
return errors
|
||||
|
||||
|
||||
def sync_all(config: Config, client: WikiClient, *, check: bool = False) -> list[str]:
|
||||
"""检查或写入所有显式映射；绝不处理映射外的文件。"""
|
||||
|
||||
if not check:
|
||||
dirty = dirty_mirror_paths(config)
|
||||
if dirty:
|
||||
details = "\n".join(dirty)
|
||||
raise WikiDocsError(
|
||||
"已映射的本地镜像存在未提交改动，已停止以防覆盖：\n" + details
|
||||
)
|
||||
|
||||
messages: list[str] = []
|
||||
for mapping in config.mappings:
|
||||
page = client.get_page(mapping.page)
|
||||
target = ROOT / PurePosixPath(mapping.path)
|
||||
if check:
|
||||
errors = check_mirror(mapping, page, target)
|
||||
if errors:
|
||||
raise WikiDocsError("\n".join(errors))
|
||||
messages.append(f"一致：{mapping.path} <- {page.title}@{page.revision[:12]}")
|
||||
continue
|
||||
existing = target.read_text(encoding="utf-8") if target.is_file() else None
|
||||
rendered = render_mirror(page, existing)
|
||||
if existing != rendered:
|
||||
_write_atomic(target, rendered)
|
||||
messages.append(f"已更新：{mapping.path} <- {page.title}@{page.revision[:12]}")
|
||||
else:
|
||||
messages.append(f"无变化：{mapping.path} <- {page.title}@{page.revision[:12]}")
|
||||
return messages
|
||||
|
||||
|
||||
def append_mapping(config: Config, mapping: Mapping) -> None:
|
||||
raw = json.loads(config.path.read_text(encoding="utf-8"))
|
||||
mappings = validate_mappings(raw.get("mappings"))
|
||||
if any(item.page == mapping.page or item.path == mapping.path for item in mappings):
|
||||
raise WikiDocsError(f"页面或路径已经登记：{mapping.page} -> {mapping.path}")
|
||||
raw["mappings"].append({"page": mapping.page, "path": mapping.path})
|
||||
rendered = json.dumps(raw, ensure_ascii=False, indent=2) + "\n"
|
||||
_write_atomic(config.path, rendered)
|
||||
Reference in New Issue
Block a user