From bf3ac6f66458c18e87ede58848d0aacdee47ef9d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 17 Nov 2026 01:58:48 +0800 Subject: [PATCH] fix(isc): terminate truncated maps; name malformed escape errors Two WO20 differential fuzz fixes (campaign: interscript/maps#284): - interscript-py#43 (hang): the two _stage block loops broke only on `}`, so a map truncated before the closing brace span forever in the skip-line fallback. Add the standard EOF progress invariant (the _block_items guard); the trailing _expect("}") names the failure with line:col. The new sweep test parses every truncation of two verbatim corpus maps and requires a document or a named IscParseError - never a spin, never an accidental exception. - interscript-py#44 (the campaign's largest crash class, 107 cases): malformed \u escapes leaked a raw ValueError from int(), including from metadata note text, which shares this scanner. Decoding now routes through _unicode_escape and raises IscParseError with line:col at the escape. The capital \U form stays literal-U (the ts hand-parser parity deviation; decoding both spellings is the owner-held cross-runtime alignment). --- src/interscript/isc.py | 43 ++++++-- tests/test_isc.py | 233 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 8 deletions(-) diff --git a/src/interscript/isc.py b/src/interscript/isc.py index f1e147e..50f2199 100644 --- a/src/interscript/isc.py +++ b/src/interscript/isc.py @@ -249,7 +249,12 @@ def _stage(self) -> dict: body: list[dict] = [] while True: self._skip_ws() - if self._peek() == "}": + # Progress invariant (same guard as _block_items): EOF means + # the block never closes — break and let the trailing + # _expect("}") name the failure. Pre-fix the skip-line + # fallback below span forever on truncated maps + # (interscript-py#43, the WO20 fuzz campaign's hang class). + if self._peek() == "}" or self._pos >= len(self._src): break if self._peek() == "#": self._skip_line() @@ -262,7 +267,8 @@ def _stage(self) -> dict: rules = [] while True: self._skip_ws() - if self._peek() == "}": + # Same EOF progress invariant as the stage loop. + if self._peek() == "}" or self._pos >= len(self._src): break if self._peek() == "#": self._skip_line() @@ -534,15 +540,35 @@ def _quoted_string(self) -> str: nxt = self._src[self._pos] if self._pos < len(self._src) else "" self._pos += 1 if nxt == "u": - hex4 = self._src[self._pos : self._pos + 4] - self._pos += 4 - result += chr(int(hex4, 16)) + result += self._unicode_escape() else: result += {"n": "\n", "t": "\t", "r": "\r"}.get(nxt, nxt) else: result += c self._error("Unterminated string") + def _unicode_escape(self) -> str: + """Decode ``\\uXXXX`` — the grammar's esc_uni4 (exactly 4 hex + digits, ``grammar/isc.parg``). Any other form (short hex, + non-hex digits, the braced form, end of input) is a parse error + at the escape's position: the pipeline used to leak a raw + ``ValueError`` from ``int()`` here (interscript-py#44, the WO20 + fuzz campaign's largest crash class — it fires on note text too, + which shares this scanner). + + The capital ``\\U`` form is deliberately NOT decoded by this + parser: the grammar's esc_u_lone branch renders it as a literal + "U" with the digits as raw text — the same deviation ts carries + in its actions.ts (ruby's artifact parser decodes both + spellings; aligning the runtimes is an owner decision). + """ + slash = self._pos - 2 # the backslash, for the error position + digits = self._src[self._pos : self._pos + 4] + if len(digits) < 4 or any(c not in "0123456789abcdefABCDEF" for c in digits): + self._error("Invalid \\u escape: expected exactly 4 hexadecimal digits", slash) + self._pos += 4 + return chr(int(digits, 16)) + def _single_quoted(self) -> str: self._expect("'") start = self._pos @@ -641,9 +667,10 @@ def _is_item_atom_start(self) -> bool: c = self._peek() return c in ('"', "'") or c.isalnum() or c == "_" - def _error(self, message: str) -> None: - line = self._src.count("\n", 0, self._pos) + 1 - col = self._pos - (self._src.rfind("\n", 0, self._pos) + 1) + 1 + def _error(self, message: str, pos: int | None = None) -> None: + at = self._pos if pos is None else pos + line = self._src.count("\n", 0, at) + 1 + col = at - (self._src.rfind("\n", 0, at) + 1) + 1 raise IscParseError(message, line, col, self._filename) diff --git a/tests/test_isc.py b/tests/test_isc.py index 4b83e78..3d937ad 100644 --- a/tests/test_isc.py +++ b/tests/test_isc.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import re from pathlib import Path import pytest @@ -286,3 +287,235 @@ def test_library_aliases_resolve_intra_library_references(): al = _library_aliases("var-kor", libs) assert "jamo_leading_cons" not in al["jamo"] assert "\\u1100" in al["jamo"] or "ᄀ" in al["jamo"] + + +# --------------------------------------------------------------------------- +# WO20 differential fuzz regressions — interscript-py#43 (parse hang) and +# interscript-py#44 (raw ValueError on malformed \u escapes). +# --------------------------------------------------------------------------- + + +def _parse_only(source: str, filename: str | None = None) -> dict: + """The parse layer only. Both fuzz issues live in the parser; running + conversion would conflate them with the engine's documented + UnsupportedConstruct vocabulary errors.""" + from interscript.isc import _IscParser + + return _IscParser(source, filename).parse() + + +# -- interscript-py#43: truncated maps must terminate with IscParseError -- + + +def test_truncated_stage_unterminated_sub_raises(): + """The campaign's 10-line repro: EOF inside `stage main` after a + complete `sub { }` block, with the stage and system braces + unterminated. Pre-fix this spun in _stage's skip-line fallback + forever (the loop broke only on `}`).""" + src = ( + 'system "VAR:hin-Deva:Latn:hunterian-1872" {\n' + "metadata {\n" + " notes {\n" + " }\n" + "}\n" + "tests {\n" + "}\n" + "stage main {\n" + " sub {\n" + " }\n" + ) + with pytest.raises(IscParseError): + _parse_only(src, "hang.isc") + + +def test_truncated_sequence_rule_block_raises(): + """Same class, inner loop: EOF inside a sequence/parallel rule block + after a complete rule — the inner loop's skip-line fallback had the + identical missing-EOF guard.""" + with pytest.raises(IscParseError): + _parse_only('system "x" { stage main { sequence { sub "a" "b"') + + +def test_truncated_map_prefixes_terminate_named(): + """Class closure: every truncation of a real corpus map terminates + with a document or a named IscParseError — never a spin, never an + accidental ValueError/TypeError. The fixtures are real corpus maps, + committed verbatim (var-ara-Arab-Arab-rababa, royin-tha-Thai-Latn- + 1999; dependency-free so the full text parses).""" + fixtures = { + "var-ara-Arab-Arab-rababa.isc": _RABABA_MAP, + "royin-tha-Thai-Latn-1999.isc": _ROYIN_THA_MAP, + } + for name, text in fixtures.items(): + # The full map parses (the sweep is meaningless otherwise). + _parse_only(text, name) + for cut in range(len(text)): + try: + _parse_only(text[:cut], name) + except IscParseError: + continue + + +_RABABA_MAP = '''system "VAR:ara-Arab:Arab:rababa" { + +metadata { + authority_id var + id rababa + language iso-639-2:ara + source_script Arab + destination_script Arab + name RABABA the Arabic Diacritization Library + url https://github.com/interscript/rababa/ + description { Arabic diacritization is useful for several practical business cases + like text to speech or Romanization of Arabic texts or scripts. + + This map in particular uses the RABABA library to automatically detect + and add diacritics using machine learning. + } +} + +tests { + 'قطر' -> 'قِطْرَ' + 'abc' -> 'abc' + '‘Iz. Ibrāhīm as-Sa‘danī' -> '‘Iz. Ibrāhīm as-Sa‘danī' +} + +stage main { + rababa config: "200" +} +} +''' + +_ROYIN_THA_MAP = '''system "ROYIN:tha-Thai:Latn:1999" { + +metadata { + authority_id royin + id 1999 + language iso-639-2:tha + source_script Thai + destination_script Latn + name Royal Thai General System of Transcription (1999) — phoneme-to-RTGS stage + url http://www.royin.go.th/wp-content/uploads/royin-ebook/276/FileUpload/758_6484.pdf + creation_date 1999 + adoption_date 2002 + description + + notes { + "This is the PHONEME-to-RTGS tail stage of the Thai composite: it consumes a syllable-dotted phonemic reading (the output of a trained Thai reading layer, e.g. sequitur/ByT5 over the pythainlp thai-g2p corpus or the tha-phonemic lexicon) and renders the Royal Thai General System of Transcription. + + Historical: the v1 maps corpus carried this mapping as the post-rules of a seq2seq-dependent map (transcription: sequitur.pythainlp_lexicon); revived here as a standalone stage for the B6 composite (WO67). + + Tones (˨˩˦˧˥) and the syllable separator (.) are not romanized (the RTGS drops tone; the separator marks syllable bounds)." + } +} + +tests { + "k a ˨˩ . k aː ˧" -> "kaka" + "k o ˨˩ . h a n ˩˩˦" -> "kohan" + "k o k̚ ˨˩ . t͡ɕʰ aː ŋ ˦˥" -> "kokchang" + "r a ˦˥ . d u ŋ ˧" -> "radung" +} + + +stage main { + + # RTGS POST-RULES (revived from maps-staging/royin-tha-Thai-Latn-1999, + # the seq2seq composite tail; the phoneme inventory is the pythainlp + # thai-g2p / royalin one) + parallel { + sub "t͡ɕʰ" "ch" + sub "t͡ɕ" "ch" + sub "ʰ" "h" + sub "ŋ" "ng" + sub "j" "i" + sub "w" "w" + sub "ɔ" "o" + sub "ɤ" "oe" + sub "ɛ" "ae" + sub "ɯ" "ue" + sub "ʔ" "" + sub "̚" "" + sub "ː" "" + sub "˨" "" + sub "˩" "" + sub "˦" "" + sub "˧" "" + sub "˥" "" + sub "." "" + sub " " "" + } +} +} +''' + + +# -- interscript-py#44: malformed \u escapes name the error -- + + +def test_malformed_escape_issue_repro_raises_named(): + """The issue's 90-byte repro: the braced form `\\u{1F600}` is not a + grammar escape (esc_uni4 wants exactly 4 hex). Pre-fix this leaked a + raw ValueError from int().""" + src = ( + 'system "T:fz-em6:1" {\n' + " stage main {\n" + " sequence {\n" + ' sub "\\u{1F600}" "X"\n' + " }\n" + " }\n" + "}\n" + ) + with pytest.raises(IscParseError): + isc_to_tree(src, "escape.isc") + + +def test_malformed_escape_matrix(): + """Every malformed lowercase \\u form is an IscParseError with + position info; note text is scanned too (the issue's note-text + finding); valid escapes still decode.""" + def expect_named_error(src: str) -> None: + with pytest.raises(IscParseError) as excinfo: + _parse_only(src) + # Position pointer present (the IscParseError renders line:col). + assert re.search(r"line \d+:\d+: ", str(excinfo.value)) + + system = 'system "x" { stage main { sub %s "X" } }' + # short hex (3 digits, then the closing quote) + expect_named_error(system % '"\\u00E"') + # non-hex head and tail + expect_named_error(system % '"\\uZZZZ"') + expect_named_error(system % '"\\u12G4"') + # braced form + expect_named_error(system % '"\\u{1F600}"') + # escape cut by end-of-string / end-of-file + expect_named_error(system % '"\\u00"') + expect_named_error('system "x" { stage main { sub "\\u') + + # note text: the escape scanner runs over free note strings too + expect_named_error( + 'system "x" {\nmetadata {\n notes {\n note "\\uZZZZ"\n }\n}\n}' + ) + + +def test_valid_escape_still_decodes(): + """The fix is error-path only: a well-formed \\uXXXX decodes exactly + as before.""" + doc = _parse_only('system "x" { stage main { sub "\\u00E9" "X" } }') + rule = doc["stages"][0]["body"][0]["rule"] + assert rule["from"] == {"type": "string", "value": "é"} + + +def test_capital_escape_stays_literal_u(): + """Capital \\U is NOT decoded by this parser: the grammar's esc_u_lone + branch renders a literal "U" and the digits stay raw text (the ts + hand-parser parity deviation in actions.ts; ruby's artifact parser + decodes both spellings — cross-runtime alignment is an owner + decision). Malformed \\U therefore never raises; it must also never + crash (#44's adjacent shape).""" + system = 'system "x" { stage main { sub %s "X" } }' + doc = _parse_only(system % '"\\UZZZZ"') + rule = doc["stages"][0]["body"][0]["rule"] + assert rule["from"] == {"type": "string", "value": "UZZZZ"} + doc = _parse_only(system % '"\\U00E"') + rule = doc["stages"][0]["body"][0]["rule"] + assert rule["from"] == {"type": "string", "value": "U00E"}