From b9b61ca827d11b9eb627b57e74a08d75ad609289 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 16 Nov 2026 21:43:29 +0800 Subject: [PATCH 1/5] WO20: r7 -> tiny student scan + distill HF job (microkimi recipe) train_wo20_tiny_hf.py - two-stage, kill-gated. Stage scan: verify the r7 teacher release (teachers-arabic-v1, per-part sha256) and the canonical labels (teacher_labels_r7.jsonl, e70ce991), capture teacher/student decoder streams on shared teacher-forced positions, closed-form directional ridge-R2 over every (student, teacher) layer pair with scrambled controls, pre-registered kill rule (R2 >= 0.30 and >= 2x control), low-rank frozen bridges on arm. Stage distill: the 2.0 rung verbatim (Muon 0.01 + AdamW 1e-4, sequence KD alpha 0.5 T 2.0, BS 8, 3 epochs, labels <=384 B/side) plus the single delta beta*MSE through the frozen bridges; step checkpoints every 500 with labels.sha resume; epilogue decodes student AND in-job teacher on the full 1,200-paragraph SadeedDiac-25 windowed zero-skip protocol and emits the paired bootstrap (documents = resampling unit) + final_eval.json. wo20_muon.py: verbatim copy of the lineage optimizer (interscript-models attic/gpu/muon.py) so the job /code mount carries the exact 2.0 routing. --- train_wo20_tiny_hf.py | 707 ++++++++++++++++++++++++++++++++++++++++++ wo20_muon.py | 199 ++++++++++++ 2 files changed, 906 insertions(+) create mode 100644 train_wo20_tiny_hf.py create mode 100644 wo20_muon.py diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py new file mode 100644 index 0000000..0138667 --- /dev/null +++ b/train_wo20_tiny_hf.py @@ -0,0 +1,707 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "torch==2.5.1", +# "transformers==4.46.3", +# "accelerate>=1.1.0", +# "pandas", +# "pyarrow", +# "pyarabic", +# "prettytable", +# ] +# /// +"""WO20 campaign A — r7 -> tiny student via the microkimi recipe (issue #441). + +Stage scan (WO20_STAGE=scan, t4/a10g): + downloads + verifies the r7 teacher (release teachers-arabic-v1, + per-part sha256) and the canonical labels (teacher_labels_r7.jsonl, + sha e70ce991), captures decoder hidden streams of teacher and + byt5-small student on the same teacher-forced positions, runs the + closed-form directional ridge-R2 scan over every (student layer, + teacher layer) pairing, applies the PRE-REGISTERED kill rule + (best R2 >= 0.30 AND >= 2x its scrambled control), and when armed + solves the low-rank bridges (ridge W, SVD-truncated, bias refit) + for the best teacher layer per student layer. Writes + bridges.npz + scan.json to the bucket. Kill = the negative result; + no distillation is armed. + +Stage distill (WO20_STAGE=distill, a100-large; requires bridges.npz): + the 2.0 rung verbatim (byt5-small, Muon lr 0.01 + AdamW 1e-4 group, + sequence KD alpha 0.5 T 2.0, BS 8, 3 epochs, labels <= 384 B/side, + cosine warmup total//20, clip 1.0, step ckpts every 500 + resume) + plus the single pre-registered delta: beta * MSE through the frozen + bridges at the scan-selected pairs. Epilogue (never fails the run): + full 1,200-paragraph SadeedDiac-25 windowed zero-skip decode for + BOTH student and in-job teacher, per-paragraph DER counts, paired + bootstrap vs r7 (documents = resampling unit), final_eval.json. + +Usage: + hf jobs uv run --flavor t4-small -d --timeout 4h \\ + -e WO20_STAGE=scan -e WO20_RUN=wo20-tiny-scan \\ + --secrets HF_TOKEN \\ + -v hf://buckets/Interscript/isx-training:/ckpt:rw \\ + -v /tmp/isx-wo20-code:/code:ro \\ + train_wo20_tiny_hf.py + + hf jobs uv run --flavor a100-large -d --timeout 4h \\ + -e WO20_STAGE=distill -e WO20_RUN=wo20-tiny-g2w \\ + --secrets HF_TOKEN \\ + -v hf://datasets/Interscript/arabic-r8:/train_data:ro \\ + -v hf://buckets/Interscript/isx-training:/ckpt:rw \\ + -v /tmp/isx-wo20-code:/code:ro \\ + train_wo20_tiny_hf.py +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import urllib.request +import zipfile +from pathlib import Path + +import numpy as np + +sys.path.insert(0, "/code") + +RUN = os.environ.get("WO20_RUN", "wo20-tiny") +STAGE = os.environ.get("WO20_STAGE", "scan") +CKPT = Path("/ckpt") / RUN +WORK = Path("/tmp/wo20") + +RELEASE = ("https://github.com/interscript/interscript-models/" + "releases/download/teachers-arabic-v1") +TEACHER_PARTS = ["run-007-news.zip.part-00", "run-007-news.zip.part-01"] +LABELS_FILE = "teacher_labels_r7.jsonl" +LABELS_SHA = None # pinned from teachers-sha256.txt at fetch time +STUDENT_INIT = "google/byt5-small" +TEACHER_TOK = "google/byt5-base" + +PAIRS_MAX_BYTES = 384 # the 2.0 recipe's Pairs filter, verbatim +BS = 8 +EPOCHS = int(os.environ.get("WO20_EPOCHS", "3")) +ALPHA = float(os.environ.get("WO20_ALPHA", "0.5")) +TEMP = float(os.environ.get("WO20_TEMP", "2.0")) +BETA = float(os.environ.get("WO20_BETA", "0.1")) +BRIDGE_RANK = 256 +REL_LAMBDA = 1e-4 +CAL_POSITIONS = 30000 # per-side cap on captured decoder positions +KILL_BAR = 0.30 # pre-registered: TODO.sota-2026/06 +KILL_MARGIN = 2.0 +WINDOW = 1400 # the published windowed protocol budget +SAVE_EVERY = 500 + +DIACRITICS_RE = __import__("re").compile("[ؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۨ-ۭ]") + + +def log(*a): + print(f"[wo20:{STAGE}]", *a, flush=True) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fetch(url: str, dest: Path) -> Path: + if dest.exists() and dest.stat().st_size > 0: + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + log(f"GET {url}") + with urllib.request.urlopen(url) as r, dest.open("wb") as f: + while True: + chunk = r.read(1 << 20) + if not chunk: + break + f.write(chunk) + return dest + + +def fetch_teacher_and_labels() -> tuple[Path, Path]: + """Download + verify the r7 teacher zip (parts) and the labels. + Digests pinned from the release's own teachers-sha256.txt.""" + manifest = {} + for line in fetch(f"{RELEASE}/teachers-sha256.txt", + WORK / "teachers-sha256.txt").read_text().splitlines(): + if line.strip(): + digest, name = line.split(None, 1) + manifest[name.strip()] = digest + for part in TEACHER_PARTS: + expect = manifest[part] + got = sha256_file(fetch(f"{RELEASE}/{part}", WORK / part)) + if got != expect: + raise SystemExit(f"sha mismatch: {part} {got} != {expect}") + log(f"verified {part}") + zpath = WORK / "run-007-news.zip" + if not zpath.exists(): + with zpath.open("wb") as out: + for part in TEACHER_PARTS: + out.write((WORK / part).read_bytes()) + tdir = WORK / "run-007-news" / "best" + if not tdir.exists(): + with zipfile.ZipFile(zpath) as zf: + zf.extractall(WORK) + global LABELS_SHA + LABELS_SHA = manifest[LABELS_FILE] + labels = fetch(f"{RELEASE}/{LABELS_FILE}", WORK / LABELS_FILE) + got = sha256_file(labels) + if got != LABELS_SHA: + raise SystemExit(f"sha mismatch: {LABELS_FILE} {got} != {LABELS_SHA}") + log(f"verified {LABELS_FILE} sha {LABELS_SHA[:12]}") + return tdir, labels + + +class Gram: + """Chunk-accumulated rectangular Gram sufficient statistics (the + microkimi scan's mergeable-by-addition statistics).""" + + def __init__(self, dx: int, dy: int): + self.dx, self.dy = dx, dy + self.gxx = np.zeros((dx, dx), np.float64) + self.gxd = np.zeros((dx, dy), np.float64) + self.gdd = 0.0 + self.n = 0 + + def add(self, X: np.ndarray, D: np.ndarray): + X = np.ascontiguousarray(X, np.float64) + D = np.ascontiguousarray(D, np.float64) + self.gxx += X.T @ X + self.gxd += X.T @ D + self.gdd += float((D * D).sum()) + self.n += len(X) + + def solve(self) -> tuple[np.ndarray, float]: + lam = REL_LAMBDA * (np.trace(self.gxx) / max(self.dx, 1)) + W = np.linalg.solve(self.gxx + lam * np.eye(self.dx), self.gxd) + res = (self.gdd - 2.0 * float((W * self.gxd).sum()) + + float(np.einsum("ij,ik,kj->", W, self.gxx, W))) + return W, 1.0 - res / max(self.gdd, 1e-30) + + +def load_pairs(labels_path: Path) -> list[tuple[str, str]]: + rows = [] + for line in labels_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + s = (row.get("src") or "").strip() + t = (row.get("tgt") or row.get("teacher") or "").strip() + if s and t and len(s.encode()) <= PAIRS_MAX_BYTES \ + and len(t.encode()) <= PAIRS_MAX_BYTES: + rows.append((s, t)) + if not rows: + raise SystemExit("no usable label pairs — check the labels file " + "schema (src + tgt|teacher)") + log(f"usable label pairs: {len(rows)}") + return rows + + +def decoder_blocks(model): + """transformers 4.x keeps the stack at model.decoder, 5.x at + model.model.decoder — read both.""" + inner = getattr(model, "model", None) + stack = getattr(inner, "decoder", None) if inner else None + if stack is None: + stack = model.decoder + return stack.block + + +def capture_streams(teacher, student, tok, pairs): + """Teacher-forced decoder hidden streams at shared byte positions.""" + import torch + from torch.utils.data import DataLoader, Dataset + + class P(Dataset): + def __len__(self): + return len(pairs) + + def __getitem__(self, i): + return pairs[i] + + def collate(batch): + src = tok([s for s, _ in batch], padding=True, return_tensors="pt") + labels = tok([t for _, t in batch], padding=True, + return_tensors="pt").input_ids + labels[labels == tok.pad_token_id] = -100 + return src.input_ids, src.attention_mask, labels + + nl_s = len(decoder_blocks(student)) + nl_t = len(decoder_blocks(teacher)) + ds, dt = student.config.d_model, teacher.config.d_model + streams_s = [np.zeros((0, ds), np.float32) for _ in range(nl_s + 1)] + streams_t = [np.zeros((0, dt), np.float32) for _ in range(nl_t + 1)] + loader = DataLoader(P(), batch_size=BS, shuffle=False, + collate_fn=collate, num_workers=2) + with torch.no_grad(): + for ids, am, labels in loader: + ids, am, labels = ids.cuda(), am.cuda(), labels.cuda() + mask = labels != -100 + if not bool(mask.any()): + continue + tout = teacher(input_ids=ids, attention_mask=am, labels=labels, + output_hidden_states=True) + sout = student(input_ids=ids, attention_mask=am, labels=labels, + output_hidden_states=True) + for k in range(nl_s + 1): + take = sout.decoder_hidden_states[k].float()[mask] + if len(take): + streams_s[k] = np.concatenate( + [streams_s[k], take.cpu().numpy()]) + for l in range(nl_t + 1): + take = tout.decoder_hidden_states[l].float()[mask] + if len(take): + streams_t[l] = np.concatenate( + [streams_t[l], take.cpu().numpy()]) + if len(streams_s[0]) >= CAL_POSITIONS: + break + streams_s = [s[:CAL_POSITIONS] for s in streams_s] + streams_t = [s[:CAL_POSITIONS] for s in streams_t] + if len(streams_s[0]) < 1000: + raise SystemExit(f"capture too small ({len(streams_s[0])} positions) " + "— refusing to solve a degenerate Gram") + log(f"captured {len(streams_s[0])} positions: student {nl_s + 1} " + f"streams ({ds}d), teacher {nl_t + 1} streams ({dt}d)") + return streams_s, streams_t + + +def run_scan(): + import torch + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + + CKPT.mkdir(parents=True, exist_ok=True) + tdir, labels_path = fetch_teacher_and_labels() + tok = AutoTokenizer.from_pretrained(STUDENT_INIT) + teacher = AutoModelForSeq2SeqLM.from_pretrained( + tdir, attn_implementation="eager").cuda().eval().half() + student = AutoModelForSeq2SeqLM.from_pretrained(STUDENT_INIT).cuda().eval() + for p in teacher.parameters(): + p.requires_grad_(False) + for p in student.parameters(): + p.requires_grad_(False) + + pairs = load_pairs(labels_path) + streams_s, streams_t = capture_streams(teacher, student, tok, pairs) + del teacher + torch.cuda.empty_cache() + + n_s, n_t = len(streams_s), len(streams_t) + scores = np.zeros((n_s, n_t), np.float64) + chunk = 8192 + for k in range(n_s): + X = streams_s[k] + for l in range(n_t): + D = streams_t[l] + g = Gram(X.shape[1], D.shape[1]) + for t0 in range(0, min(len(X), len(D)), chunk): + g.add(X[t0:t0 + chunk], D[t0:t0 + chunk]) + _, scores[k, l] = g.solve() + log(f"scan R2 student L{k} -> teacher L{l}: {scores[k, l]:.4f}") + + # scrambled control: same solve against row-permuted teacher streams + rng = np.random.default_rng(7) + armed, controls = [], {} + for k in range(n_s): + l = int(np.argmax(scores[k])) + D = streams_t[l].copy() + X = streams_s[k] + g = Gram(X.shape[1], D.shape[1]) + perm = rng.permutation(len(D)) + for t0 in range(0, min(len(X), len(D)), chunk): + g.add(X[t0:t0 + chunk], D[perm[t0:t0 + chunk]]) + _, ctrl = g.solve() + controls[f"L{k}:L{l}"] = ctrl + log(f"control student L{k} -> teacher L{l}: {ctrl:.4f} " + f"(real {scores[k, l]:.4f})") + armed.append((k, l)) + + best = float(scores.max()) + best_pair = tuple(int(v) for v in np.unravel_index(scores.argmax(), + scores.shape)) + best_ctrl = controls.get(f"L{best_pair[0]}:L{best_pair[1]}", 0.0) + verdict = ("ARMED" if best >= KILL_BAR + and best >= KILL_MARGIN * max(best_ctrl, 1e-9) else "KILLED") + log(f"best pair L{best_pair[0]}->L{best_pair[1]}: R2={best:.4f} " + f"control={best_ctrl:.4f} -> {verdict}") + + bridges_path = CKPT / "bridges.npz" + if verdict == "ARMED": + payload, meta = {}, [] + for k, l in armed: + X, D = streams_s[k], streams_t[l] + g = Gram(X.shape[1], D.shape[1]) + for t0 in range(0, min(len(X), len(D)), chunk): + g.add(X[t0:t0 + chunk], D[t0:t0 + chunk]) + W, r2 = g.solve() + U, S, Vt = np.linalg.svd(W, full_matrices=False) + r = min(BRIDGE_RANK, len(S)) + # W (d_s x d_t) = B @ A with B (d_s x r), A (r x d_t); + # apply as pred = (h_s @ B) @ A + bias + B = U[:, :r] * S[:r] + A = Vt[:r] + bias = D.mean(0).astype(np.float64) - X.mean(0) @ W + payload[f"B_L{k}"] = B.astype(np.float32) + payload[f"A_L{k}"] = A.astype(np.float32) + payload[f"bias_L{k}"] = bias.astype(np.float32) + meta.append({"student_layer": k, "teacher_layer": l, + "r2": float(r2), "rank": r}) + log(f"bridge L{k}->L{l}: r2={r2:.4f} rank={r}") + np.savez(bridges_path, **payload) + scan = { + "stage": "scan", + "run": RUN, + "scores": scores.tolist(), + "controls": controls, + "armed_pairs": [list(p) for p in armed], + "best_pair": list(best_pair), + "best_r2": best, + "best_control": best_ctrl, + "kill_bar": KILL_BAR, + "kill_margin": KILL_MARGIN, + "verdict": verdict, + "labels_sha256": LABELS_SHA, + "cal_positions": int(len(streams_s[0])), + "student_layers": n_s, + "teacher_layers": n_t, + } + (CKPT / "scan.json").write_text(json.dumps(scan, indent=1)) + (CKPT / "final_eval.json").write_text(json.dumps( + {"stage": "scan", "run": RUN, "verdict": verdict, + "best_pair": list(best_pair), "best_r2": best, + "best_control": best_ctrl, "labels_sha256": LABELS_SHA}, indent=1)) + log(f"scan complete: {verdict} -> {bridges_path if verdict == 'ARMED' else '(no bridges)'}") + + +def run_distill(): + import torch + import torch.nn.functional as F + from torch.utils.data import DataLoader, Dataset + from transformers import (AutoModelForSeq2SeqLM, AutoTokenizer, + get_cosine_schedule_with_warmup) + from wo20_muon import Muon, split_parameters + + CKPT.mkdir(parents=True, exist_ok=True) + bridges_file = CKPT / "bridges.npz" + if not bridges_file.exists(): + raise SystemExit("bridges.npz missing — run the scan stage first " + "(the kill rule gates this stage)") + scan = json.loads((CKPT / "scan.json").read_text()) + if scan.get("verdict") != "ARMED": + raise SystemExit(f"scan verdict is {scan.get('verdict')} — " + "distillation is not armed") + npz = np.load(bridges_file) + pairs_meta = scan["armed_pairs"] + + tdir, labels_path = fetch_teacher_and_labels() + tok = AutoTokenizer.from_pretrained(STUDENT_INIT) + teacher = AutoModelForSeq2SeqLM.from_pretrained( + tdir, attn_implementation="eager").cuda().eval().half() + for p in teacher.parameters(): + p.requires_grad_(False) + student = AutoModelForSeq2SeqLM.from_pretrained(STUDENT_INIT).cuda() + student.train() + + # frozen bridges: pred_t = (h_s @ B) @ A + bias + bridges = [] + for meta in pairs_meta: + k = meta["student_layer"] + B = torch.from_numpy(npz[f"B_L{k}"]).cuda() + A = torch.from_numpy(npz[f"A_L{k}"]).cuda() + bias = torch.from_numpy(npz[f"bias_L{k}"]).cuda() + bridges.append((k, meta["teacher_layer"], B, A, bias)) + for _, _, B, A, _ in bridges: + B.requires_grad_(False) + A.requires_grad_(False) + + rows = load_pairs(labels_path) + + class P(Dataset): + def __len__(self): + return len(rows) + + def __getitem__(self, i): + return rows[i] + + def collate(batch): + src = tok([s for s, _ in batch], padding=True, return_tensors="pt") + labels = tok([t for _, t in batch], padding=True, + return_tensors="pt").input_ids + labels[labels == tok.pad_token_id] = -100 + return src.input_ids, src.attention_mask, labels + + loader = DataLoader(P(), batch_size=BS, shuffle=True, collate_fn=collate, + num_workers=2, drop_last=True) + total_steps = len(loader) * EPOCHS + muon_params, adamw_params = split_parameters( + list(student.named_parameters())) + optimizer = Muon(muon_params, lr=0.01, momentum=0.95, weight_decay=0.01) + optimizer.add_adamw_group(adamw_params, lr=1e-4, weight_decay=0.0) + log(f"optimizer: muon {len(muon_params)} matrix / " + f"adamw {len(adamw_params)} embedding-like params") + scheduler = get_cosine_schedule_with_warmup(optimizer, + total_steps // 20, total_steps) + + labels_digest = LABELS_SHA[:12] if LABELS_SHA else "none" + + def _usable(ck: Path) -> bool: + marker = ck / "labels.sha" + return (marker.exists() + and marker.read_text().strip() == labels_digest) + + ckpts = sorted((c for c in CKPT.glob("step-*") if _usable(c)), + key=lambda p: int(p.name.split("-")[1])) + step = 0 + if ckpts: + student.load_state_dict(torch.load(ckpts[-1] / "student.pt", + map_location="cuda", + weights_only=True)) + optimizer.load_state_dict(torch.load(ckpts[-1] / "optim.pt", + map_location="cuda", + weights_only=True)) + step = int(ckpts[-1].name.split("-")[1]) + for _ in range(step): + scheduler.step() + log(f"resume from {ckpts[-1].name}") + + for old in ckpts[:-2]: + for f in old.iterdir(): + f.unlink() + old.rmdir() + + def save(step_no): + ck = CKPT / f"step-{step_no}" + ck.mkdir(exist_ok=True) + torch.save(student.state_dict(), ck / "student.pt") + torch.save(optimizer.state_dict(), ck / "optim.pt") + (ck / "labels.sha").write_text(labels_digest) + + log(f"train: {len(rows)} pairs x {EPOCHS} epochs = {total_steps} steps, " + f"alpha={ALPHA} T={TEMP} beta={BETA} " + f"bridges={[(k, l) for k, l, *_ in bridges]}") + + while step < total_steps: + for ids, am, labels in loader: + if step >= total_steps: + break + ids, am, labels = ids.cuda(), am.cuda(), labels.cuda() + with torch.no_grad(): + tout = teacher(input_ids=ids, attention_mask=am, labels=labels, + output_hidden_states=True) + s_out = student(input_ids=ids, attention_mask=am, labels=labels, + output_hidden_states=True) + mask = labels != -100 + kd = F.kl_div( + F.log_softmax(s_out.logits[mask] / TEMP, dim=-1), + F.softmax(tout.logits.float()[mask] / TEMP, dim=-1), + reduction="batchmean") * (TEMP ** 2) + loss = ALPHA * kd + (1 - ALPHA) * s_out.loss + if bridges: + mse_terms = [] + for k, l, B, A, bias in bridges: + h_s = s_out.decoder_hidden_states[k][mask].float() + h_t = tout.decoder_hidden_states[l][mask].float() + pred = (h_s @ B.float() @ A.float()) + bias.float() + mse_terms.append(F.mse_loss(pred, h_t)) + bridge_mse = torch.stack(mse_terms).mean() + loss = loss + BETA * bridge_mse + else: + bridge_mse = torch.zeros((), device=ids.device) + loss.backward() + torch.nn.utils.clip_grad_norm_(student.parameters(), 1.0) + optimizer.step() + scheduler.step() + optimizer.zero_grad() + step += 1 + if step % 50 == 0: + log(f"step {step}/{total_steps} loss={float(loss):.4f} " + f"ce={float(s_out.loss):.4f} kd={float(kd):.4f} " + f"mse={float(bridge_mse):.4f}") + if step % SAVE_EVERY == 0: + save(step) + + out_dir = CKPT / "best" + out_dir.mkdir(exist_ok=True) + student.save_pretrained(out_dir) + tok.save_pretrained(out_dir) + log(f"final student saved -> {out_dir}") + run_eval(student, tok, teacher) + + +def split_windows(text: str, budget: int = WINDOW) -> list[str]: + if len(text.encode("utf-8")) <= budget: + return [text] + words, cur, n, wins = text.split(), [], 0, [] + for w in words: + c = len(w.encode("utf-8")) + 1 + if cur and n + c > budget: + wins.append(" ".join(cur)) + cur, n = [], 0 + cur.append(w) + n += c + if cur: + wins.append(" ".join(cur)) + return wins + + +def paired_bootstrap(der_a: np.ndarray, der_b: np.ndarray, + counts: np.ndarray, boot: int = 10000, + seed: int = 7): + """Documents as resampling unit (the microkimi eval_compare math, + count-weighted). Returns (delta = b - a, lo, hi, p_delta_gt_0).""" + keep = counts > 0 + a = np.asarray(der_a, np.float64)[keep] * counts[keep] + b = np.asarray(der_b, np.float64)[keep] * counts[keep] + c = counts[keep].astype(np.float64) + delta = b.sum() / c.sum() - a.sum() / c.sum() + rng = np.random.default_rng(seed) + n = len(c) + idx = rng.integers(0, n, size=(boot, n)) + da = a[idx].sum(axis=1) + db = b[idx].sum(axis=1) + ct = c[idx].sum(axis=1) + deltas = (db - da) / ct + lo, hi = np.percentile(deltas, [2.5, 97.5]) + p = float((deltas > 0).mean()) + return float(delta), float(lo), float(hi), p + + +def run_eval(student, tok, teacher): + """Full 1,200-paragraph windowed zero-skip protocol for the student + AND the in-job teacher; paired bootstrap vs r7.""" + import pyarrow.parquet as pq + import torch + from sadeed_evaluator import ArabicDiacritizationEvaluator as E + + teacher.eval() + student.eval() + + table = pq.read_table(Path("/train_data") / "sadeeddiac-25.parquet") + inputs = [DIACRITICS_RE.sub("", t) for t in table.column("input").to_pylist()] + outputs = table.column("output").to_pylist() + log(f"eval paragraphs: {len(inputs)}") + + windows, counts = [], [] + for text in inputs: + ws = split_windows(text) + counts.append(len(ws)) + windows.extend(ws) + + def decode_all(model, name, prog_name, autocast_bf16: bool): + prog = CKPT / prog_name + saved = {} + if prog.exists(): + for line in prog.read_text(encoding="utf-8").splitlines(): + if line.strip(): + row = json.loads(line) + saved[row["i"]] = row["pred"] + missing = [i for i in range(len(windows)) if i not in saved] + model.eval() + bs = 8 + with torch.no_grad(): + for t0 in range(0, len(missing), bs): + chunk_ids = missing[t0:t0 + bs] + batch = [windows[i] for i in chunk_ids] + enc = tok(batch, return_tensors="pt", padding=True, + truncation=True, max_length=WINDOW).to("cuda") + ctx = (torch.autocast("cuda", torch.bfloat16) + if autocast_bf16 else torch.autocast("cuda", + enabled=False)) + with ctx: + gen = model.generate( + **enc, + max_new_tokens=int(2 * enc.input_ids.shape[1]), + num_beams=1) + preds = tok.batch_decode(gen, skip_special_tokens=True) + for i, p in zip(chunk_ids, preds): + saved[i] = p + with prog.open("a", encoding="utf-8") as out: + for i, p in zip(chunk_ids, preds): + out.write(json.dumps({"i": i, "pred": p}, + ensure_ascii=False) + "\n") + if (t0 // bs) % 25 == 0: + log(f"[{name}] {len(saved)}/{len(windows)}") + k = 0 + paras = [] + for c in counts: + paras.append(" ".join(saved[i] for i in range(k, k + c))) + k += c + return paras + + student_paras = decode_all(student, "student", + "eval_progress_student.jsonl", True) + (CKPT / "student_preds.jsonl").write_text( + "\n".join(json.dumps({"src": i, "student": p}, ensure_ascii=False) + for i, p in enumerate(student_paras))) + teacher_paras = decode_all(teacher, "teacher", + "eval_progress_teacher.jsonl", False) + (CKPT / "teacher_preds.jsonl").write_text( + "\n".join(json.dumps({"src": i, "teacher": p}, ensure_ascii=False) + for i, p in enumerate(teacher_paras))) + + ev = E() + gt = list(outputs) + n = len(gt) + s_der = np.zeros(n) + t_der = np.zeros(n) + letters = np.zeros(n) + s_wer = np.zeros(n) + t_wer = np.zeros(n) + words = np.zeros(n) + for i in range(n): + wc, lc, _, sw, _, sd, _ = ev.caculate_error_on_single_sentence( + student_paras[i], gt[i], gt_missing_diacritic_is_error=False) + _, _, _, tw, _, td, _ = ev.caculate_error_on_single_sentence( + teacher_paras[i], gt[i], gt_missing_diacritic_is_error=False) + words[i], letters[i] = wc, max(lc, 1) + s_wer[i], t_wer[i] = sw, tw + s_der[i] = 100.0 * sd / max(lc, 1) + t_der[i] = 100.0 * td / max(lc, 1) + + s_total = float(100.0 * (s_der * letters).sum() / letters.sum()) + t_total = float(100.0 * (t_der * letters).sum() / letters.sum()) + delta, lo, hi, p_worse = paired_bootstrap(t_der, s_der, letters) + t_wer_total = float(100.0 * (t_wer * words).sum() / max(words.sum(), 1)) + s_wer_total = float(100.0 * (s_wer * words).sum() / max(words.sum(), 1)) + gate = t_total + 0.5 + verdict = ("GATE-FAIL" if s_total > gate else "GATE-PASS") + result = { + "stage": "distill", + "run": RUN, + "recipe": {"alpha": ALPHA, "temperature": TEMP, "beta": BETA, + "epochs": EPOCHS, "bs": BS, + "pairs_max_bytes": PAIRS_MAX_BYTES, + "student_init": STUDENT_INIT, + "labels_sha256": LABELS_SHA}, + "teacher_full_der_ce": t_total, + "student_full_der_ce": s_total, + "student_full_wer": s_wer_total, + "teacher_full_wer": t_wer_total, + "gate_teacher_plus_halfpp": gate, + "paired_delta_pp_student_minus_teacher": delta, + "paired_ci95": [lo, hi], + "p_student_worse": p_worse, + "verdict": verdict, + "n_paragraphs": n, + } + log("RESULT " + json.dumps(result)) + (CKPT / "final_eval.json").write_text(json.dumps(result, indent=1)) + + +def main(): + if STAGE == "scan": + run_scan() + elif STAGE == "distill": + run_distill() + else: + raise SystemExit(f"unknown WO20_STAGE: {STAGE}") + + +if __name__ == "__main__": + main() diff --git a/wo20_muon.py b/wo20_muon.py new file mode 100644 index 0000000..f8aac00 --- /dev/null +++ b/wo20_muon.py @@ -0,0 +1,199 @@ +"""Muon optimizer (single file): orthogonalized momentum via +Newton–Schulz for 2D weight matrices, with AdamW-fallback groups. + +From Keller Jordan's Muon (modded-nanogpt); the recipe Qwen3.8-Flash-Next +/ LongCat-Flash-Lite report training with. Rules of use: +- Matrices that are updated as WHOLE weights (attention/FFN projections) + get Newton–Schulz-orthogonalized momentum. +- Embedding-like tensors — byte embeddings, layer norms, the tied + lm_head, relative-attention bias, and memory-layer lookup tables + (random access per the quantization-class policy) — stay on AdamW + inside the same optimizer object, via groups flagged ``adamw=True``. + +State save/resume works through the standard ``torch.optim`` dict. +""" + +from __future__ import annotations + +import torch + + +def zeropower_via_newtonschulz5(g: torch.Tensor, steps: int = 5) -> torch.Tensor: + # quintic iteration coefficients from the modded-nanogpt lineage + a, b, c = 3.4445, -4.7750, 2.0315 + x = g.bfloat16() + x = x / (x.norm() + 1e-7) + transposed = g.size(-2) > g.size(-1) + if transposed: + x = x.mT + for _ in range(steps): + a_mat = x @ x.mT + b_mat = b * a_mat + c * (a_mat @ a_mat) + x = a * x + b_mat @ x + if transposed: + x = x.mT + return x.to(g.dtype) + + +_MUON_DEFAULTS = {"lr": 0.01, "momentum": 0.95, "nesterov": True, + "ns_steps": 5, "weight_decay": 0.0} + + +class Muon(torch.optim.Optimizer): + def __init__(self, params, lr: float = 0.01, momentum: float = 0.95, + nesterov: bool = True, ns_steps: int = 5, + weight_decay: float = 0.0) -> None: + settings = {**_MUON_DEFAULTS, "lr": lr, "momentum": momentum, + "nesterov": nesterov, "ns_steps": ns_steps, + "weight_decay": weight_decay, "adamw": False} + super().__init__(list(params), settings) + + def add_adamw_group(self, params, lr: float = 1e-4, betas=(0.9, 0.999), + weight_decay: float = 0.0) -> None: + """Embedding-like parameters: standard AdamW math, shared + scheduler (the cosine scales every group's lr).""" + self.add_param_group({ + "params": list(params), + "lr": lr, + "betas": tuple(betas), + "weight_decay": weight_decay, + "adamw": True, + }) + + def add_headwise_group(self, params, heads: int) -> None: + """Per-head preconditioning (head-wise Muon): each attention + head's rows of a Q/K weight are orthogonalized as their own + matrix. Accepts params already inside a Muon group (their + group is converted in place) or all-new params (a new group is + added with the constructor's default settings).""" + params = list(params) + owned = {id(p) for group in self.param_groups for p in group["params"]} + hits = {id(p) for p in params if id(p) in owned} + if hits and len(hits) != len(params): + raise ValueError("headwise routing takes all-new or all-existing params") + if hits: + for group in self.param_groups: + members = {id(q) for q in group["params"]} + if hits & members: + if hits != members: + raise ValueError( + "headwise routing converts whole groups; " + "split the group first") + group["headwise"] = True + group["heads"] = heads + return + settings = {**_MUON_DEFAULTS, "adamw": False, + "params": params, "headwise": True, "heads": heads} + self.add_param_group(settings) + + @torch.no_grad() + def step(self, closure=None): # noqa: ARG002 + for group in self.param_groups: + if group.get("adamw"): + self._adamw_step(group) + elif group.get("headwise"): + self._muon_step_headwise(group) + else: + self._muon_step(group) + + def _momentum_direction(self, p, group): + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(p.grad) + buf = state["momentum_buffer"] + buf.lerp_(p.grad, 1 - group["momentum"]) + return p.grad.lerp(buf, group["momentum"]) if group["nesterov"] else buf + + def _muon_step(self, group) -> None: + for p in group["params"]: + if p.grad is None: + continue + g = self._momentum_direction(p, group) + u = zeropower_via_newtonschulz5(g, steps=group["ns_steps"]) + if group["weight_decay"]: + p.mul_(1 - group["lr"] * group["weight_decay"]) + p.add_(u.to(p.dtype), alpha=-group["lr"] * max(1, p.size(-2) / p.size(-1)) ** 0.5) + + def _muon_step_headwise(self, group) -> None: + heads = group["heads"] + for p in group["params"]: + if p.grad is None: + continue + if p.size(0) % heads: + raise ValueError(f"param rows {p.size(0)} not divisible by {heads} heads") + g = self._momentum_direction(p, group) + if group["weight_decay"]: + p.mul_(1 - group["lr"] * group["weight_decay"]) + d = p.size(0) // heads + for h in range(heads): + rows = slice(h * d, (h + 1) * d) + u = zeropower_via_newtonschulz5(g[rows], steps=group["ns_steps"]) + scale = max(1, g[rows].size(-2) / g[rows].size(-1)) ** 0.5 + p[rows].add_(u.to(p.dtype), alpha=-group["lr"] * scale) + + def _adamw_step(self, group) -> None: + beta1, beta2 = group["betas"] + for p in group["params"]: + if p.grad is None: + continue + state = self.state[p] + if "adamw_exp_avg" not in state: + state["adamw_exp_avg"] = torch.zeros_like(p.grad) + state["adamw_exp_avg_sq"] = torch.zeros_like(p.grad) + state["adamw_step"] = 0 + exp_avg, exp_avg_sq = state["adamw_exp_avg"], state["adamw_exp_avg_sq"] + state["adamw_step"] += 1 + exp_avg.lerp_(p.grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(p.grad, p.grad, value=1 - beta2) + bias_c1 = 1 - beta1 ** state["adamw_step"] + bias_c2 = 1 - beta2 ** state["adamw_step"] + denom = (exp_avg_sq / bias_c2).sqrt_().add_(1e-8) + if group["weight_decay"]: + p.mul_(1 - group["lr"] * group["weight_decay"]) + p.addcdiv_(exp_avg / bias_c1, denom, value=-group["lr"]) + + +def _is_embedding_like(name: str, p) -> bool: + return ( + p.ndim < 2 + or "embed_tokens" in name + or name == "shared.weight" # T5 tied byte embedding (transformers 5.x) + or "lm_head" in name + or "relative_attention" in name + or "memory.values" in name + or "memory.k1" in name + or "memory.k2" in name + ) + + +def embed_named(named_params): + """The 2D embedding-like tensors (tied embedding, prediction heads, + memory tables) — SinkhornUpdate's surface: rows are token identity, + columns features. 1D params (layer norms) are excluded; they have + no row/column structure to balance.""" + return [ + (name, p) for name, p in named_params + if p.requires_grad and p.ndim >= 2 and _is_embedding_like(name, p) + ] + + +def split_parameters(named_params): + """The standard split: orthogonalizable 2D hidden weights vs + embedding-like tensors (1D params, embeddings, tied head, relative + bias, memory lookup tables).""" + muon, adamw = [], [] + for name, p in named_params: + if not p.requires_grad: + continue + (adamw if _is_embedding_like(name, p) else muon).append(p) + return muon, adamw + + +def qk_named(named_params): + """The Q/K projection weights head-wise Muon applies to (DeepSeek + V4.1 Flash sec 2.5; GLM-5 and Kimi-K3 validate the same split). + T5 names both self- and cross-attention projections.""" + import re + + pattern = re.compile(r"(SelfAttention|EncDecAttention)\.(q|k)\.weight$") + return [(name, p) for name, p in named_params if pattern.search(name)] From e73fbc6c1b9a210a9296adb50c2bd758c32e47fb Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 16 Nov 2026 21:46:41 +0800 Subject: [PATCH 2/5] WO20 distill: read scan artifacts from the scan run dir (WO20_SCAN_RUN) --- train_wo20_tiny_hf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py index 0138667..81630b4 100644 --- a/train_wo20_tiny_hf.py +++ b/train_wo20_tiny_hf.py @@ -68,7 +68,9 @@ RUN = os.environ.get("WO20_RUN", "wo20-tiny") STAGE = os.environ.get("WO20_STAGE", "scan") +SCAN_RUN = os.environ.get("WO20_SCAN_RUN", "wo20-tiny-scan") CKPT = Path("/ckpt") / RUN +SCAN_CKPT = Path("/ckpt") / SCAN_RUN WORK = Path("/tmp/wo20") RELEASE = ("https://github.com/interscript/interscript-models/" @@ -388,11 +390,11 @@ def run_distill(): from wo20_muon import Muon, split_parameters CKPT.mkdir(parents=True, exist_ok=True) - bridges_file = CKPT / "bridges.npz" + bridges_file = SCAN_CKPT / "bridges.npz" if not bridges_file.exists(): raise SystemExit("bridges.npz missing — run the scan stage first " "(the kill rule gates this stage)") - scan = json.loads((CKPT / "scan.json").read_text()) + scan = json.loads((SCAN_CKPT / "scan.json").read_text()) if scan.get("verdict") != "ARMED": raise SystemExit(f"scan verdict is {scan.get('verdict')} — " "distillation is not armed") From 01c95b81eb4824d132dfcc8e8bba21f8b2f78b10 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 16 Nov 2026 21:49:16 +0800 Subject: [PATCH 3/5] WO20 distill: prune old step checkpoints at save time (keep last 2) --- train_wo20_tiny_hf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py index 81630b4..d35acdd 100644 --- a/train_wo20_tiny_hf.py +++ b/train_wo20_tiny_hf.py @@ -483,6 +483,12 @@ def save(step_no): torch.save(student.state_dict(), ck / "student.pt") torch.save(optimizer.state_dict(), ck / "optim.pt") (ck / "labels.sha").write_text(labels_digest) + keep = sorted(CKPT.glob("step-*"), + key=lambda p: int(p.name.split("-")[1])) + for old in keep[:-2]: + for f in old.iterdir(): + f.unlink() + old.rmdir() log(f"train: {len(rows)} pairs x {EPOCHS} epochs = {total_steps} steps, " f"alpha={ALPHA} T={TEMP} beta={BETA} " From d416fd69cb32b73f21108444bce0a7892079984c Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 16 Nov 2026 21:52:28 +0800 Subject: [PATCH 4/5] WO20 distill: read armed_pairs as the scan's [k, l] list format --- train_wo20_tiny_hf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py index d35acdd..a99b8d3 100644 --- a/train_wo20_tiny_hf.py +++ b/train_wo20_tiny_hf.py @@ -413,11 +413,14 @@ def run_distill(): # frozen bridges: pred_t = (h_s @ B) @ A + bias bridges = [] for meta in pairs_meta: - k = meta["student_layer"] + if isinstance(meta, dict): + k, l = meta["student_layer"], meta["teacher_layer"] + else: + k, l = int(meta[0]), int(meta[1]) B = torch.from_numpy(npz[f"B_L{k}"]).cuda() A = torch.from_numpy(npz[f"A_L{k}"]).cuda() bias = torch.from_numpy(npz[f"bias_L{k}"]).cuda() - bridges.append((k, meta["teacher_layer"], B, A, bias)) + bridges.append((k, l, B, A, bias)) for _, _, B, A, _ in bridges: B.requires_grad_(False) A.requires_grad_(False) From 2f9e54a6b9d76f9b1970a58aae307961653ae05b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 16 Nov 2026 21:56:26 +0800 Subject: [PATCH 5/5] WO20 distill: correct the recipe data filter to the lineage's actual caps The ara specs cap units at max_len 1450 B and labels at 2x max_len = 2900 B (accept_label) with collate truncation at those caps and gradient checkpointing on - my 384-B default came from a different Pairs call site and produced 112 usable pairs instead of the lineage's ~29k. Fix the caps, the collate truncation, and disable the cache/checkpointing for the eval decode. --- train_wo20_tiny_hf.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/train_wo20_tiny_hf.py b/train_wo20_tiny_hf.py index a99b8d3..0663121 100644 --- a/train_wo20_tiny_hf.py +++ b/train_wo20_tiny_hf.py @@ -81,7 +81,8 @@ STUDENT_INIT = "google/byt5-small" TEACHER_TOK = "google/byt5-base" -PAIRS_MAX_BYTES = 384 # the 2.0 recipe's Pairs filter, verbatim +PAIRS_MAX_BYTES = 1450 # the ara spec's max_len: units <= 1450 B +LABEL_CAP_BYTES = 2900 # the lineage's label cap = 2 x max_len BS = 8 EPOCHS = int(os.environ.get("WO20_EPOCHS", "3")) ALPHA = float(os.environ.get("WO20_ALPHA", "0.5")) @@ -197,7 +198,7 @@ def load_pairs(labels_path: Path) -> list[tuple[str, str]]: s = (row.get("src") or "").strip() t = (row.get("tgt") or row.get("teacher") or "").strip() if s and t and len(s.encode()) <= PAIRS_MAX_BYTES \ - and len(t.encode()) <= PAIRS_MAX_BYTES: + and len(t.encode()) <= LABEL_CAP_BYTES: rows.append((s, t)) if not rows: raise SystemExit("no usable label pairs — check the labels file " @@ -408,6 +409,8 @@ def run_distill(): for p in teacher.parameters(): p.requires_grad_(False) student = AutoModelForSeq2SeqLM.from_pretrained(STUDENT_INIT).cuda() + student.gradient_checkpointing_enable() + student.config.use_cache = False student.train() # frozen bridges: pred_t = (h_s @ B) @ A + bias @@ -435,8 +438,10 @@ def __getitem__(self, i): return rows[i] def collate(batch): - src = tok([s for s, _ in batch], padding=True, return_tensors="pt") - labels = tok([t for _, t in batch], padding=True, + src = tok([s for s, _ in batch], padding=True, truncation=True, + max_length=PAIRS_MAX_BYTES, return_tensors="pt") + labels = tok([t for _, t in batch], padding=True, truncation=True, + max_length=LABEL_CAP_BYTES, return_tensors="pt").input_ids labels[labels == tok.pad_token_id] = -100 return src.input_ids, src.attention_mask, labels @@ -542,6 +547,8 @@ def save(step_no): student.save_pretrained(out_dir) tok.save_pretrained(out_dir) log(f"final student saved -> {out_dir}") + student.gradient_checkpointing_disable() + student.config.use_cache = True run_eval(student, tok, teacher)