brainwave/scripts/import-zhouyi-wikisource.mjs

import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { kingWenEntries } from "./content-contract.mjs";
import { repositoryRoot } from "./repository-files.mjs";

const execFileAsync = promisify(execFile);
const proxyUrl = process.env.HTTPS_PROXY || process.env.HTTP_PROXY || "http://127.0.0.1:1080";
const limit = Number.parseInt(process.env.IMPORT_LIMIT ?? "64", 10);
const userAgent = "LingjiContentImport/1.0 (local Zhouyi pipeline; public-domain jingwen only)";
const outputDirectory = path.join(repositoryRoot, "content", "raw");
const outputPath = path.join(outputDirectory, "zhouyi-wikisource-jing.json");

const titles = [
  "乾", "坤", "屯", "蒙", "需", "訟", "師", "比",
  "小畜", "履", "泰", "否", "同人", "大有", "謙", "豫",
  "隨", "蠱", "臨", "觀", "噬嗑", "賁", "剝", "復",
  "无妄", "大畜", "頤", "大過", "坎", "離", "咸", "恆",
  "遯", "大壯", "晉", "明夷", "家人", "睽", "蹇", "解",
  "損", "益", "夬", "姤", "萃", "升", "困", "井",
  "革", "鼎", "震", "艮", "漸", "歸妹", "豐", "旅",
  "巽", "兌", "渙", "節", "中孚", "小過", "既濟", "未濟",
];

const linePattern = /^(初九|九二|九三|九四|九五|上九|初六|六二|六三|六四|六五|上六|用九|用六)[：:，,](.*)$/u;

function stripMarkup(wikitext) {
  return wikitext
    .replace(/-\{([^}|]+)-\}/gu, "$1")
    .replace(/\{\{[^}]*\}\}/gu, "")
    .replace(/<[^>]+>/gu, "")
    .replace(/'{2,}/gu, "")
    .replace(/\[\[File:[^\]]*\]\]/gu, "")
    .replace(/\[\[(?:[^\|\]]*\|)?([^\]]+)\]\]/gu, "$1")
    .replace(/&nbsp;/gu, " ")
    .replace(/\r/gu, "");
}

function parseClassicFields(section) {
  const rawLines = section
    .split("\n")
    .map((line) => line.replace(/^[ *#:]+/u, "").trim())
    .filter(Boolean);

  let judgment = "";
  const lines = [];
  let special = null;
  for (const rawLine of rawLines) {
    const compact = rawLine.replace(/\s+/gu, "");
    const hit = linePattern.exec(compact) ?? linePattern.exec(rawLine);
    if (hit) {
      const label = hit[1];
      const text = `${label}：${hit[2].trim()}`;
      if (label === "用九" || label === "用六") {
        special = text;
      } else {
        lines.push(text);
      }
    } else if (lines.length === 0 && special == null) {
      const fragment = rawLine.replace(/\s+/gu, "");
      judgment = judgment ? `${judgment}${fragment}` : fragment;
    }
  }

  judgment = judgment
    .replace(/^(?:周易)?[\u4e00-\u9fff]{1,3}[：:]/u, "")
    .trim();
  if (!judgment) throw new Error("missing judgment");
  if (lines.length !== 6) {
    throw new Error(`expected 6 lines, got ${lines.length}: ${lines.join(" | ")}`);
  }
  return { judgmentOriginal: judgment, lineTextsBottomUp: lines, specialUsageText: special };
}

function extractClassicSection(wikitext) {
  const normalized = stripMarkup(wikitext).replace(/\u3000/gu, " ");
  const classicIndex = normalized.search(/易[經经][：:]/u);
  if (classicIndex < 0) throw new Error("missing 易經 section");
  const afterClassic = normalized.slice(classicIndex);
  const stop = afterClassic.search(/\n[ *#]*彖曰[：:]/u);
  return (stop >= 0 ? afterClassic.slice(0, stop) : afterClassic)
    .replace(/^易[經经][：:]\s*/u, "")
    .trim();
}

function stripHtml(value) {
  return value
    .replace(/<[^>]+>/gu, "")
    .replace(/&quot;/gu, "\"")
    .replace(/&amp;/gu, "&")
    .replace(/&lt;/gu, "<")
    .replace(/&gt;/gu, ">")
    .replace(/\n+/gu, "\n")
    .trim();
}

async function curlJson(args) {
  const { stdout } = await execFileAsync("curl.exe", [
    "-sS",
    "--fail",
    "--retry",
    "12",
    "--retry-delay",
    "20",
    "--retry-all-errors",
    "--max-time",
    "45",
    "-x",
    proxyUrl,
    "-H",
    `User-Agent: ${userAgent}`,
    "-H",
    "Accept: application/json",
    ...args,
  ], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 });
  return JSON.parse(stdout);
}

const titleFallbacks = {
  恆: ["恒"],
  遯: ["遁"],
  晉: ["晋"],
  大壯: ["大壮"],
  歸妹: ["归妹"],
  豐: ["丰"],
  兌: ["兑"],
  渙: ["涣"],
  節: ["节"],
  既濟: ["既济"],
  未濟: ["未济"],
};

async function fetchWikitext(title) {
  const candidates = [title, ...(titleFallbacks[title] ?? [])];
  for (const candidate of candidates) {
    const url = new URL("https://zh.wikisource.org/w/api.php");
    url.searchParams.set("action", "parse");
    url.searchParams.set("page", `周易/${candidate}`);
    url.searchParams.set("prop", "wikitext");
    url.searchParams.set("format", "json");
    const payload = await curlJson([url.toString()]);
    const wikitext = payload?.parse?.wikitext?.["*"];
    if (typeof wikitext === "string") return wikitext;
    process.stdout.write(`missing ${candidate}, trying fallback\n`);
  }
  throw new Error(`no wikitext for ${title}`);
}

async function toHansFields(classic) {
  const pieces = [
    classic.judgmentOriginal,
    ...classic.lineTextsBottomUp,
    ...(classic.specialUsageText == null ? [] : [classic.specialUsageText]),
  ];
  const converted = await convertToHans(pieces.join("\n¶\n"));
  const parts = converted
    .split("¶")
    .map((part) => part.replace(/\s+/gu, "").trim())
    .filter(Boolean);
  if (parts.length !== pieces.length) {
    throw new Error(`zh-hans field count ${parts.length} != ${pieces.length}: ${converted}`);
  }
  return {
    judgmentOriginal: parts[0],
    lineTextsBottomUp: parts.slice(1, 7),
    specialUsageText: classic.specialUsageText == null ? null : parts[7],
  };
}
async function convertToHans(text) {
  const payload = await curlJson([
    "--data-urlencode", "action=parse",
    "--data-urlencode", `text=${text}`,
    "--data-urlencode", "prop=text",
    "--data-urlencode", "variant=zh-hans",
    "--data-urlencode", "disablelimitreport=1",
    "--data-urlencode", "wrapoutputclass=",
    "--data-urlencode", "contentmodel=wikitext",
    "--data-urlencode", "format=json",
    "https://zh.wikisource.org/w/api.php",
  ]);
  const html = payload?.parse?.text?.["*"];
  if (typeof html !== "string") throw new Error("zh-hans conversion returned no text");
  return stripHtml(html);
}

const entries = kingWenEntries();
await mkdir(outputDirectory, { recursive: true });
let imported = [];
try {
  imported = JSON.parse(await readFile(outputPath, "utf8"));
  if (!Array.isArray(imported)) imported = [];
} catch {
  imported = [];
}
const done = new Set(imported.map((item) => item.kingWenNumber));

for (const [index, title] of titles.entries()) {
  if (index >= limit) break;
  const kingWenNumber = index + 1;
  if (done.has(kingWenNumber)) {
    process.stdout.write(`skip ${kingWenNumber}/64 ${title}\n`);
    continue;
  }
  const wikitext = await fetchWikitext(title);
  const traditional = extractClassicSection(wikitext);
  const classic = await toHansFields(parseClassicFields(traditional));
  imported.push({
    kingWenNumber,
    wikisourceTitle: title,
    sourcePage: `https://zh.wikisource.org/wiki/周易/${title}`,
    ...entries[index],
    ...classic,
  });
  imported.sort((left, right) => left.kingWenNumber - right.kingWenNumber);
  await writeFile(outputPath, `${JSON.stringify(imported, null, 2)}\n`);
  process.stdout.write(`imported ${kingWenNumber}/64 ${title}\n`);
  await new Promise((resolve) => setTimeout(resolve, 1200));
}

process.stdout.write(`wrote ${outputPath} (${imported.length} hexagrams)\n`);