From 4a7d61e97fd5120a16191c55786490bc14cef4ae Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 17 Nov 2026 21:07:16 +0800 Subject: [PATCH 1/3] WO20u M1: WO20_SEED knob + the multi-seed confirmation scorer The bridge arm's separated-improvement vs G2a (-0.4385pp, p=0) is a single-seed result; the ledger's seed-variance caveat (2608.27729) applies. Seeds 2 and 3 run the identical recipe (same scan bridges, same 2.0 rung base, beta 0.1) as explicit independent draws: WO20_SEED seeds torch/numpy/python/DataLoader-workers; the unseeded seed-1 run stays as-is and is recorded as an uncontrolled draw. wo20_seed_confirm.py scores any seed's banked student preds against G2a's released preds (frontier-predictions-v1, sha-verified) with the identical evaluator + count-weighted paired bootstrap; the seed-1 calibration (must reproduce -0.4385pp CI [-0.65, -0.28]) precedes any seed-2/3 reading. Judgment pre-registered in interscript-models TODO.sota-2026/06 before launch. --- train_wo20_tiny_hf.py | 28 +++++- wo20_seed_confirm.py | 225 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 wo20_seed_confirm.py diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py index 0663121..e746f99 100644 --- a/train_wo20_tiny_hf.py +++ b/train_wo20_tiny_hf.py @@ -69,6 +69,7 @@ RUN = os.environ.get("WO20_RUN", "wo20-tiny") STAGE = os.environ.get("WO20_STAGE", "scan") SCAN_RUN = os.environ.get("WO20_SCAN_RUN", "wo20-tiny-scan") +SEED = int(os.environ.get("WO20_SEED", "0")) # 0 = unseeded (the seed-1 run) CKPT = Path("/ckpt") / RUN SCAN_CKPT = Path("/ckpt") / SCAN_RUN WORK = Path("/tmp/wo20") @@ -391,6 +392,16 @@ def run_distill(): from wo20_muon import Muon, split_parameters CKPT.mkdir(parents=True, exist_ok=True) + if SEED: + # WO20u M1 multi-seed: explicit independent draw of the identical + # recipe (seed 1 ran unseeded — entropy seed — recorded as such). + import random + + random.seed(SEED) + np.random.seed(SEED) + torch.manual_seed(SEED) + torch.cuda.manual_seed_all(SEED) + log(f"seed: WO20_SEED={SEED} (torch/numpy/python/data-worker)") bridges_file = SCAN_CKPT / "bridges.npz" if not bridges_file.exists(): raise SystemExit("bridges.npz missing — run the scan stage first " @@ -446,8 +457,20 @@ def collate(batch): labels[labels == tok.pad_token_id] = -100 return src.input_ids, src.attention_mask, labels + loader_gen = None + worker_init = None + if SEED: + def _worker_init(wid): + wseed = SEED + wid + random.seed(wseed) + np.random.seed(wseed) + torch.manual_seed(wseed) + loader_gen = torch.Generator() + loader_gen.manual_seed(SEED) + worker_init = _worker_init loader = DataLoader(P(), batch_size=BS, shuffle=True, collate_fn=collate, - num_workers=2, drop_last=True) + num_workers=2, drop_last=True, generator=loader_gen, + worker_init_fn=worker_init) total_steps = len(loader) * EPOCHS muon_params, adamw_params = split_parameters( list(student.named_parameters())) @@ -499,7 +522,7 @@ def save(step_no): old.rmdir() log(f"train: {len(rows)} pairs x {EPOCHS} epochs = {total_steps} steps, " - f"alpha={ALPHA} T={TEMP} beta={BETA} " + f"alpha={ALPHA} T={TEMP} beta={BETA} seed={SEED or 'unseeded'} " f"bridges={[(k, l) for k, l, *_ in bridges]}") while step < total_steps: @@ -694,6 +717,7 @@ def decode_all(model, name, prog_name, autocast_bf16: bool): "run": RUN, "recipe": {"alpha": ALPHA, "temperature": TEMP, "beta": BETA, "epochs": EPOCHS, "bs": BS, + "seed": SEED or "unseeded", "pairs_max_bytes": PAIRS_MAX_BYTES, "student_init": STUDENT_INIT, "labels_sha256": LABELS_SHA}, diff --git a/wo20_seed_confirm.py b/wo20_seed_confirm.py new file mode 100644 index 0000000..7ab324c --- /dev/null +++ b/wo20_seed_confirm.py @@ -0,0 +1,225 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "numpy>=1.26", +# "pandas", +# "pyarrow", +# "pyarabic", +# "prettytable", +# ] +# /// +"""WO20u M1 — multi-seed confirmation scorer (issue #441 follow-up). + +Scores one WO20 bridge-arm seed run's banked student predictions against +the G2a frontier rung (run-007-r7-muon-6ep, released predictions in +`frontier-predictions-v1`) under the IDENTICAL evaluator and bootstrap +math the seed-1 verdict used: + + - full 1,200-paragraph SadeedDiac-25, windowed zero-skip protocol + (preds are already window-joined paragraphs banked by the trainer's + eval epilogue); + - sadeed_evaluator, `gt_missing_diacritic_is_error=False`; + - word-count-mismatch paragraphs skipped PAIRWISE (the evaluator + raises on length mismatch — the skip is its aggregate behavior; + seed 1: 59 of 1,200 skipped); + - count-weighted paired bootstrap, documents = resampling unit, + 10k resamples, seed 7 — delta = seed - G2a (negative = seed better). + +Also re-scores the seed-1 banked preds when pointed at +WO20_RUN=wo20-tiny-g2w — the calibration that makes seeds 2/3 +comparable to the published −0.4385pp verdict. + +Usage (cpu-basic; resume-safe via the bucket output marker): + hf jobs uv run --flavor cpu-basic -d --timeout 4h \\ + -e WO20_RUN=wo20-tiny-g2w-s2 \\ + --secrets HF_TOKEN \\ + -v hf://datasets/Interscript/arabic-r8:/train_data:ro \\ + -v hf://buckets/Interscript/isx-training:/ckpt:rw \\ + -v /tmp/isx-wo20u-code:/code:ro \\ + wo20_seed_confirm.py +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import urllib.request +from pathlib import Path + +import numpy as np + +sys.path.insert(0, "/code") + +RUN = os.environ.get("WO20_RUN", "wo20-tiny-g2w-s2") +CKPT = Path("/ckpt") / RUN +OUT = Path("/ckpt") / "wo20u-m1" +FPV1 = ("https://github.com/interscript/interscript-models/" + "releases/download/frontier-predictions-v1") +G2A_PREDS = "run-007-r7-muon-6ep-final_preds.jsonl" +G2A_SHA = "f6f606195144e3e17fdc85eb558ae7d58e30b8c14876e7c26456575bf238e0d7" +BOOT = 10000 +BOOT_SEED = 7 + + +def log(*a): + print("[wo20u-m1]", *a, flush=True) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fetch(url: str, dest: Path) -> Path: + if dest.exists() and dest.stat().st_size > 0: + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + log(f"GET {url}") + with urllib.request.urlopen(url) as r, dest.open("wb") as f: + while True: + chunk = r.read(1 << 20) + if not chunk: + break + f.write(chunk) + return dest + + +def load_seed_preds(path: Path) -> dict[int, str]: + preds = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + i = row.get("src", row.get("idx")) + preds[int(i)] = row.get("student") or row.get("pred") or "" + return preds + + +def paired_bootstrap(der_a: np.ndarray, der_b: np.ndarray, + counts: np.ndarray, boot: int = BOOT, + seed: int = BOOT_SEED): + """Documents as resampling unit (the trainer's exact math, + count-weighted). Returns (delta = b - a, lo, hi, p_delta_gt_0).""" + keep = counts > 0 + a = np.asarray(der_a, np.float64)[keep] * counts[keep] + b = np.asarray(der_b, np.float64)[keep] * counts[keep] + c = counts[keep].astype(np.float64) + delta = b.sum() / c.sum() - a.sum() / c.sum() + rng = np.random.default_rng(seed) + n = len(c) + idx = rng.integers(0, n, size=(boot, n)) + da = a[idx].sum(axis=1) + db = b[idx].sum(axis=1) + ct = c[idx].sum(axis=1) + deltas = (db - da) / ct + lo, hi = np.percentile(deltas, [2.5, 97.5]) + p = float((deltas > 0).mean()) + return float(delta), float(lo), float(hi), p + + +def main(): + from sadeed_evaluator import ArabicDiacritizationEvaluator as E + + marker = OUT / f"{RUN}.confirm.json" + if marker.exists(): + log(f"already scored (bucket marker) — {marker}") + print(marker.read_text()) + return + + preds_path = CKPT / "student_preds.jsonl" + if not preds_path.exists(): + raise SystemExit(f"no banked student preds at {preds_path} — " + "the seed's eval epilogue has not completed") + seed_preds = load_seed_preds(preds_path) + log(f"seed preds: {len(seed_preds)} paragraphs") + + g2a_path = fetch(f"{FPV1}/{G2A_PREDS}", Path("/tmp") / G2A_PREDS) + got = sha256_file(g2a_path) + if got != G2A_SHA: + raise SystemExit(f"sha mismatch: {G2A_PREDS} {got}") + g2a_rows = {} + for line in g2a_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + i = row.get("idx", row.get("src")) + g2a_rows[int(i)] = row.get("student") or "" + log(f"G2a preds: {len(g2a_rows)} paragraphs (sha {got[:12]} verified)") + + import pyarrow.parquet as pq + table = pq.read_table(Path("/train_data") / "sadeeddiac-25.parquet") + gold = table.column("output").to_pylist() + log(f"gold paragraphs: {len(gold)}") + + ev = E() + common = sorted(set(seed_preds) & set(g2a_rows) + & set(range(len(gold)))) + der_seed = np.zeros(len(common)) + der_g2a = np.zeros(len(common)) + letters = np.zeros(len(common)) + words = np.zeros(len(common)) + n_skip_seed = n_skip_g2a = 0 + for j, i in enumerate(common): + gt = gold[i] + counts = {} + for name, pred in (("seed", seed_preds[i]), ("g2a", g2a_rows[i])): + try: + wc, lc, _, _, _, sd, _ = \ + ev.caculate_error_on_single_sentence( + pred, gt, gt_missing_diacritic_is_error=False) + counts[name] = (wc, lc, sd) + except RuntimeError: + counts[name] = None + if counts["seed"] is None: + n_skip_seed += 1 + if counts["g2a"] is None: + n_skip_g2a += 1 + if counts["seed"] is None or counts["g2a"] is None: + continue # pairwise skip on word-count mismatch + wc, lc, sd = counts["seed"] + words[j], letters[j] = wc, lc + der_seed[j] = 100.0 * sd / max(lc, 1) + wc2, lc2, sd2 = counts["g2a"] + der_g2a[j] = 100.0 * sd2 / max(lc2, 1) + + scorable = letters > 0 + n_scorable = int(scorable.sum()) + seed_total = float(100.0 * (der_seed[scorable] + * letters[scorable]).sum() + / letters[scorable].sum()) + g2a_total = float(100.0 * (der_g2a[scorable] + * letters[scorable]).sum() + / letters[scorable].sum()) + delta, lo, hi, p = paired_bootstrap(der_g2a, der_seed, letters) + separated = hi < 0 # CI entirely negative: separated improvement + + result = { + "stage": "wo20u-m1-confirm", + "run": RUN, + "protocol": ("full SadeedDiac-25, windowed zero-skip, paired " + "bootstrap vs G2a released preds " + "(frontier-predictions-v1 sha-verified), documents " + "= resampling unit, count-weighted, 10k, seed 7"), + "n_scorable_pairs": n_scorable, + "skipped_seed_wordcount_mismatch": n_skip_seed, + "skipped_g2a_wordcount_mismatch": n_skip_g2a, + "g2a_rescored_der_ce": round(g2a_total, 4), + "seed_der_ce_scorable": round(seed_total, 4), + "paired_delta_pp_seed_minus_g2a": round(delta, 4), + "paired_ci95": [round(lo, 4), round(hi, 4)], + "p_seed_worse": p, + "separated_improvement": separated, + "g2a_preds_sha256": got, + } + log("RESULT " + json.dumps(result)) + OUT.mkdir(parents=True, exist_ok=True) + marker.write_text(json.dumps(result, indent=1)) + + +if __name__ == "__main__": + main() From 09829d1be8cde5eb3942d0c08aa8fdc060793705 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 17 Nov 2026 21:20:54 +0800 Subject: [PATCH 2/3] WO20u M1 scorer: add tqdm dep (sadeed_evaluator imports it; the trainer got it transitively via transformers, the CPU scorer did not) --- wo20_seed_confirm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wo20_seed_confirm.py b/wo20_seed_confirm.py index 7ab324c..e423b36 100644 --- a/wo20_seed_confirm.py +++ b/wo20_seed_confirm.py @@ -6,6 +6,7 @@ # "pyarrow", # "pyarabic", # "prettytable", +# "tqdm", # ] # /// """WO20u M1 — multi-seed confirmation scorer (issue #441 follow-up). From 9edca8d286d7a957c9cb55ba202d13b6e2e4011d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 17 Nov 2026 21:22:54 +0800 Subject: [PATCH 3/3] WO20u M1 scorer: fix double-x100 on the aggregate DER fields (per-paragraph arrays are already pp; the paired delta/CI/p were always correct - the seed-1 calibration matched the published verdict exactly) --- wo20_seed_confirm.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wo20_seed_confirm.py b/wo20_seed_confirm.py index e423b36..9b39184 100644 --- a/wo20_seed_confirm.py +++ b/wo20_seed_confirm.py @@ -190,11 +190,13 @@ def main(): scorable = letters > 0 n_scorable = int(scorable.sum()) - seed_total = float(100.0 * (der_seed[scorable] - * letters[scorable]).sum() + # der_* arrays are already in percentage points (per-paragraph + # DER-CE); the aggregate is their letter-count-weighted mean. + seed_total = float((der_seed[scorable] + * letters[scorable]).sum() / letters[scorable].sum()) - g2a_total = float(100.0 * (der_g2a[scorable] - * letters[scorable]).sum() + g2a_total = float((der_g2a[scorable] + * letters[scorable]).sum() / letters[scorable].sum()) delta, lo, hi, p = paired_bootstrap(der_g2a, der_seed, letters) separated = hi < 0 # CI entirely negative: separated improvement