From 2c3c6111ffbe12dcb6588c29d5991c83cd8f13a9 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 11 Nov 2026 22:34:19 +0800 Subject: [PATCH] feat(WO17): py golden additivity gate + the constraint class-context port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of the improvement round, two pieces: - tests/test_golden_capture.py: the PERMANENT golden additivity test for interscript-py (mirror of rs tests/golden_capture.rs and ts test/golden-isc-wf0.test.ts): parse the sibling maps/libs corpus, compile, replay all 12,337 golden rows byte-identically; adopting manifest honored; sibling-gated skip (GOLDEN_ROOT override). Harness detection proven against a doctored golden (one flipped row -> failure names exactly that map). - The rs/ts constraint fix lands in the py runtime: compile_item gains class_ctx — constraint positions (before/not_before/after/not_after) and any() alias arguments compile in CLASS context, where an IMPORTED charset alias resolving to a string expands per-character (rs convert_class_alias; local string aliases stay sequences). Without it the ell lookbehinds compiled from a ~400-char literal sequence and could never match, silently disabling alalc-ell's medial gamma rules. 4 regression specs watched failing first (root-cause shape, from-any single char, local-alias sequence preservation, the five ell rows + the word-final gk guard through the in-process path). Verified: golden 12,337/12,337 byte-identical through the in-process py runtime (83 s; pre-fix exactly 5 divergent rows, all alalc-ell-Grek-Latn-2010); constraint/wf1/parser suites green. --- interscript-py/src/interscript/executor.py | 20 ++- interscript-py/src/interscript/items.py | 61 ++++++- interscript-py/tests/test_constraints.py | 181 ++++++++++++++++++++ interscript-py/tests/test_golden_capture.py | 123 +++++++++++++ 4 files changed, 371 insertions(+), 14 deletions(-) create mode 100644 interscript-py/tests/test_constraints.py create mode 100644 interscript-py/tests/test_golden_capture.py diff --git a/interscript-py/src/interscript/executor.py b/interscript-py/src/interscript/executor.py index 2f566136..1d657734 100644 --- a/interscript-py/src/interscript/executor.py +++ b/interscript-py/src/interscript/executor.py @@ -161,10 +161,12 @@ def _execute_sub_rule(rule: SubRule, ctx: ExecutionContext) -> None: raise MapLogicError("sub rule missing 'from'") src = compile_item(rule.from_, ctx) - before = compile_item(rule.before, ctx).re if rule.before else "" - after = compile_item(rule.after, ctx).re if rule.after else "" - not_before = compile_item(rule.not_before, ctx).re if rule.not_before else "" - not_after = compile_item(rule.not_after, ctx).re if rule.not_after else "" + # Constraints compile in CLASS context (charset aliases resolve + # per-character — mirrors rs materialize_class / convert_class_alias). + before = compile_item(rule.before, ctx, True).re if rule.before else "" + after = compile_item(rule.after, ctx, True).re if rule.after else "" + not_before = compile_item(rule.not_before, ctx, True).re if rule.not_before else "" + not_after = compile_item(rule.not_after, ctx, True).re if rule.not_after else "" parts: list[str] = [] if before: @@ -318,10 +320,12 @@ def sort_key(pair: tuple[int, SubRule]): compiled: list[tuple[SubRule, str, str, str, str, str]] = [] for _idx, r in sorted(enumerate(sub_rules), key=sort_key): src = compile_item(r.from_, ctx).re # type: ignore[arg-type] - before = compile_item(r.before, ctx).re if r.before else "" - after = compile_item(r.after, ctx).re if r.after else "" - not_before = compile_item(r.not_before, ctx).re if r.not_before else "" - not_after = compile_item(r.not_after, ctx).re if r.not_after else "" + # Constraints compile in CLASS context (charset aliases + # resolve per-character — mirrors rs materialize_class). + before = compile_item(r.before, ctx, True).re if r.before else "" + after = compile_item(r.after, ctx, True).re if r.after else "" + not_before = compile_item(r.not_before, ctx, True).re if r.not_before else "" + not_after = compile_item(r.not_after, ctx, True).re if r.not_after else "" compiled.append((r, src, before, after, not_before, not_after)) # §5.2 at pattern level: identical compiled `from` source under a # guard-applicable rule shadows the unguarded one. diff --git a/interscript-py/src/interscript/items.py b/interscript-py/src/interscript/items.py index 5dacb01e..bb382974 100644 --- a/interscript-py/src/interscript/items.py +++ b/interscript-py/src/interscript/items.py @@ -44,19 +44,49 @@ def _compiled(re_src: str, literal: str) -> CompiledItem: return CompiledItem(re=re_src, literal=literal) +def _string_charset(value: str) -> CompiledItem: + """Compile a string charset as a per-character SET (class context). + Mirrors rs convert_class_alias expanding Item::Str into one entry + per char, and Ruby Any#to_regexp building a char class from string + data. Compiled as an alternation (not a [...] class) so astral code + points and regexp-escaped chars stay correct.""" + chars = list(value) + return _compiled( + "(?:" + "|".join(regexp_escape(c) for c in chars) + ")", + chars[0] if chars else "", + ) + + def _alias_compiled(name: str) -> _StdlibAlias: return _StdlibAlias(re=ALIAS_COMPILED[name].re, literal=ALIAS_COMPILED[name].literal) -def compile_item(item: Item, ctx: ExecutionContext) -> CompiledItem: - """Compile an Item into regex source + literal replacement text.""" +def compile_item( + item: Item, ctx: ExecutionContext, class_ctx: bool = False +) -> CompiledItem: + """Compile an Item into regex source + literal replacement text. + + ``class_ctx`` selects the compile context (rs parity, WO17 — + mirrors builder.rs materialize_class vs materialize): + + - sequence context (default): an alias resolving to a string is a + literal sequence; + - class context (constraint positions — before/not_before/after/ + not_after — and the arguments of ``any()``): an IMPORTED charset + alias (a dependency map's alias) resolving to a string expands + one entry per character (rs convert_class_alias). A LOCAL string + alias stays a sequence even in class context. + """ kind = item.kind if kind == "string": assert isinstance(item, StringItem) return _compiled(regexp_escape(item.value), item.value) if kind == "capture_group": assert isinstance(item, CaptureGroupItem) - inner = compile_item(item.data, ctx) + # rs converts capture innards in sequence context even inside a + # constraint (convert_class_fragment only class-resolves the + # top-level alias). + inner = compile_item(item.data, ctx, False) return _compiled(f"({inner.re})", inner.literal) if kind == "capture_ref": assert isinstance(item, CaptureRefItem) @@ -69,14 +99,33 @@ def compile_item(item: Item, ctx: ExecutionContext) -> CompiledItem: resolved = ctx.resolve_alias(item.name) if resolved is None: return _compiled("", "") - return compile_item(resolved, ctx) + if ( + class_ctx + and item.name not in ctx.map.aliases + and resolved.kind == "string" + ): + # Imported charset alias in class position: per-char set. + assert isinstance(resolved, StringItem) + return _string_charset(resolved.value) + return compile_item(resolved, ctx, class_ctx) if kind == "any": assert isinstance(item, AnyItem) - parts = [compile_item(child, ctx).re for child in item.of] + # any() arguments are class-resolved regardless of the + # surrounding context (rs convert_any_arg routes alias entries + # through convert_set_entry -> convert_class_alias). + # Concatenation/capture entries stay sequence-resolved — only + # alias entries differ per context. + parts = [ + compile_item(child, ctx, child.kind == "alias").re for child in item.of + ] # Mirror Ruby's Any#nth_string: the literal form picks the first # alternative (sub rules with `to: any("ie")` use it as the # canonical replacement). - first = compile_item(item.of[0], ctx).literal if item.of else "" + first = ( + compile_item(item.of[0], ctx, item.of[0].kind == "alias").literal + if item.of + else "" + ) return _compiled("(?:" + "|".join(parts) + ")", first) if kind == "any_char_class": assert isinstance(item, AnyCharClassItem) diff --git a/interscript-py/tests/test_constraints.py b/interscript-py/tests/test_constraints.py new file mode 100644 index 00000000..245d7c72 --- /dev/null +++ b/interscript-py/tests/test_constraints.py @@ -0,0 +1,181 @@ +"""Contextual (before/after-constrained) rule regression specs — the +Python mirror of interscript-ts ``test/isc/constraints.test.ts``. + +Root-cause regression (WO17, 2026-11): an IMPORTED charset alias (a +dependency map's alias resolving to a plain string, e.g. unicode's +``greek``) used in a class position — a constraint (before/ +not_before/after/not_after) or an ``any()`` argument — compiled as a +literal SEQUENCE of the whole charset instead of a per-character SET. +The resulting lookbehind/lookahead could never match, silently +disabling every constrained rule built on it (the golden showed the +alalc-ell-Grek-Latn-2010 medial γκ rules never firing; the five +divergent rows are γκέγκε, Λαγκαδάς, Παγκράτι, Κονγκό, Νταγκεστάν). + +RS parity (interscript-rs builder.rs): + - convert_class_alias: imported Item::Str charsets expand one entry + per char in class context; LOCAL string aliases stay sequences; + - convert_any_arg: alias entries class-resolve everywhere; + - constraints materialize in class context (materialize_class); + from/to materialize in sequence context. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from interscript.executor import execute_stage +from interscript.isc import isc_to_compiled_map, parse_isc + +LIB_CODE = "py-constraint-lib" +LIB_SRC = 'system "py-constraint-lib" {\naliases {\n vowels = "aeiou"\n upper = "KQV"\n}\n}\n' + +ROOT = Path(__file__).resolve().parents[3] +MAPS_DIR = ROOT / "maps" / "maps" +LIBS_DIR = ROOT / "maps" / "libs" + + +class _Resolver: + def __init__(self, maps: dict[str, object]) -> None: + self._maps = maps + + def load(self, system_code: str): + return self._maps[system_code] + + +def _compile(sources: dict[str, str]): + """Parse + compile a dict of .isc sources with lazy dependency + resolution; returns (by_code, resolver).""" + + def resolve_dependency(target: str): + return docs.get(target) + + docs: dict[str, object] = {} + for code, src in sources.items(): + docs[code] = parse_isc(src, filename=code, resolve_dependency=resolve_dependency) + compiled = {code: isc_to_compiled_map(doc) for code, doc in docs.items()} + return compiled, _Resolver(compiled) + + +@pytest.fixture() +def corpus_available() -> None: + if not MAPS_DIR.is_dir(): + pytest.skip("sibling maps checkout missing") + + +def _run(compiled, resolver, code: str, text: str) -> str: + return execute_stage(compiled[code], "main", text, resolver) + + +def test_before_imported_charset_matches_single_char() -> None: + compiled, resolver = _compile( + { + LIB_CODE: LIB_SRC, + "py-constraint-class-ctx": ( + 'system "py-constraint-class-ctx" {\n' + f'dependency "{LIB_CODE}"\n' + "stage main {\n" + " sub {\n" + ' from "x"\n' + ' to "X"\n' + " before any(vowels)\n" + " }\n" + "}\n" + "}\n" + ), + } + ) + code = "py-constraint-class-ctx" + # before = lookbehind: the set char must PRECEDE the match. + assert _run(compiled, resolver, code, "ax") == "aX" + assert _run(compiled, resolver, code, "ux") == "uX" + # "q" is not in the set — with the sequence bug this rule could + # NEVER fire at all; here it must fire on set membership only. + assert _run(compiled, resolver, code, "qx") == "qx" + + +def test_from_imported_charset_matches_single_char() -> None: + compiled, resolver = _compile( + { + LIB_CODE: LIB_SRC, + "py-constraint-from-set": ( + 'system "py-constraint-from-set" {\n' + f'dependency "{LIB_CODE}"\n' + "stage main {\n" + " sub {\n" + " from any(vowels)\n" + ' to "-"\n' + " }\n" + "}\n" + "}\n" + ), + } + ) + code = "py-constraint-from-set" + # from any(charset) matches ONE set char (megaregexp path; with the + # sequence bug the from compiled as the whole charset literal blob). + assert _run(compiled, resolver, code, "abc") == "-bc" + + +def test_local_string_alias_stays_sequence_in_class_position() -> None: + compiled, resolver = _compile( + { + "py-constraint-local-alias": ( + 'system "py-constraint-local-alias" {\n' + "aliases {\n" + ' cluster = "th"\n' + "}\n" + "stage main {\n" + " sub {\n" + ' from "s"\n' + ' to "S"\n' + " before cluster\n" + " }\n" + "}\n" + "}\n" + ) + } + ) + code = "py-constraint-local-alias" + # LOCAL string aliases stay SEQUENCES even in class context (rs + # parity): before cluster = lookbehind for the literal "th". + assert _run(compiled, resolver, code, "ths") == "thS" + assert _run(compiled, resolver, code, "qs") == "qs" + + +def _build_sibling_corpus(): + if not MAPS_DIR.is_dir(): + pytest.skip("sibling maps checkout missing") + + def resolve_dependency(target: str): + return docs.get(target) + + docs: dict[str, object] = {} + for directory in (LIBS_DIR, MAPS_DIR): + for path in sorted(directory.glob("*.isc")): + docs[path.stem] = parse_isc( + path.read_text(encoding="utf-8"), + filename=str(path), + resolve_dependency=resolve_dependency, + ) + compiled = {stem: isc_to_compiled_map(doc) for stem, doc in docs.items()} + return compiled, _Resolver(compiled) + + +def test_ell_medial_gamma_rows_through_the_full_path() -> None: + """The five alalc-ell-Grek-Latn-2010 golden rows this bug silenced, + through the in-process parse-execute path.""" + compiled, resolver = _build_sibling_corpus() + code = "alalc-ell-Grek-Latn-2010" + cases = { + "γκέγκε": "gkenke", + "Λαγκαδάς": "Lankadas", + "Παγκράτι": "Pankrati", + "Κονγκό": "Konnko", + "Νταγκεστάν": "Ḏankestan", + # word-final γκ must keep its "gk" + "Χονγκ Κονγκ": "Chongk Kongk", + } + for src, expected in cases.items(): + assert _run(compiled, resolver, code, src) == expected, src diff --git a/interscript-py/tests/test_golden_capture.py b/interscript-py/tests/test_golden_capture.py new file mode 100644 index 00000000..6dd9fd9a --- /dev/null +++ b/interscript-py/tests/test_golden_capture.py @@ -0,0 +1,123 @@ +"""ISC-WF-0 golden capture additivity — the Python mirror of +interscript-rs ``tests/golden_capture.rs`` and interscript-ts +``test/golden-isc-wf0.test.ts``. + +Replays every row of the RS-captured golden file +(``interscript-rs/tests/golden/isc-wf0.jsonl``) through THIS runtime — +parse the sibling maps/libs ``.isc`` corpus, compile, execute — and +requires byte-identical output on every row (rows of maps listed in +the adopting manifest ``tests/golden/adopting-maps.txt`` are exempt). +The golden is the cross-runtime contract: a delta here is a runtime +regression (or a newly-fixed divergence being adopted into the +golden). + +Skips when the sibling interscript-rs / maps checkouts are missing +(``GOLDEN_ROOT`` overrides the sibling root, mirroring the ts test). +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from interscript.executor import execute_stage +from interscript.isc import isc_to_compiled_map, parse_isc + +REPO_ROOT = Path(__file__).resolve().parents[3] +SIBLING_ROOT = Path(os.environ.get("GOLDEN_ROOT", str(REPO_ROOT))) +GOLDEN = SIBLING_ROOT / "interscript-rs" / "tests" / "golden" / "isc-wf0.jsonl" +ADOPTING = SIBLING_ROOT / "interscript-rs" / "tests" / "golden" / "adopting-maps.txt" +MAPS_DIR = SIBLING_ROOT / "maps" / "maps" +LIBS_DIR = SIBLING_ROOT / "maps" / "libs" + + +def _sibling_checkouts_present() -> bool: + return GOLDEN.is_file() and MAPS_DIR.is_dir() + + +def _load_adopting() -> set[str]: + if not ADOPTING.is_file(): + return set() + return { + line.strip() + for line in ADOPTING.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + } + + +def _build_corpus() -> tuple[dict[str, object], object]: + """Parse + compile every corpus .isc; dependencies (and WF-1 + ``use alias.`` expansion) resolve lazily against the same + corpus. libs first: the parser validates ``dependency`` targets + eagerly, and system maps declare dependencies on the libraries + (e.g. thai-latn).""" + docs: dict[str, object] = {} + + def resolve_dependency(target: str): + return docs.get(target) + + for directory in (LIBS_DIR, MAPS_DIR): + for path in sorted(directory.glob("*.isc")): + docs[path.stem] = parse_isc( + path.read_text(encoding="utf-8"), + filename=str(path), + resolve_dependency=resolve_dependency, + ) + compiled = {stem: isc_to_compiled_map(doc) for stem, doc in docs.items()} + + class _Resolver: + """MapResolver over the in-memory corpus (the MapLoader protocol: + .load(system_code)).""" + + def load(self, system_code: str): + return compiled[system_code] + + return compiled, _Resolver() + + +@pytest.mark.skipif( + not _sibling_checkouts_present(), + reason="skip: golden capture or sibling maps checkout missing", +) +def test_golden_capture_additivity() -> None: + rows = [ + json.loads(line) + for line in GOLDEN.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert rows, "golden file has no rows" + compiled, resolver = _build_corpus() + adopting = _load_adopting() + + deltas: list[str] = [] + failed_maps: set[str] = set() + exempt = 0 + for row in rows: + map_id = row["map"] + if map_id in adopting: + exempt += 1 + continue + compiled_map = compiled.get(map_id) + if compiled_map is None: + got = "\x00error:map not found" + else: + try: + got = execute_stage(compiled_map, "main", row["src"], resolver) + except Exception as exc: # noqa: BLE001 — mirror the ts/rs gates + got = f"\x00error:{exc}" + if got == row["out"]: + continue + failed_maps.add(map_id) + if len(deltas) < 10: + deltas.append( + f"{map_id}: {row['src']!r} -> {got!r} (golden {row['out']!r})" + ) + + assert not failed_maps, ( + f"golden divergence on {len(failed_maps)} map(s); " + f"first deltas:\n" + "\n".join(deltas) + ) + assert len(rows) - exempt > 0