Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 226 additions & 0 deletions app/core/array_reductions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
"""Bounded typed reduction pass; scopes are supplied by structured validation."""
from dataclasses import dataclass
import json


class ReductionError(ValueError):
def __init__(self, code, message):
self.code = code
super().__init__(message)


@dataclass
class Budget:
remaining: int = 2_000_000

def charge(self, amount):
if amount > self.remaining:
raise ReductionError("resource_limit", "Shared reduction work budget exhausted")
self.remaining -= amount


@dataclass(frozen=True)
class Token:
kind: str
text: str
start: int
end: int


@dataclass(frozen=True)
class Reference:
name: str
selectors: tuple | None = None


@dataclass(frozen=True)
class Selection:
names: tuple
reduced_axes: tuple


@dataclass(frozen=True)
class Reduction:
op: str
reference: Reference
selection: Selection
start: int
end: int


@dataclass(frozen=True)
class Lowered:
source: str
equation: str
reductions: tuple
replacements: tuple


def ident(text):
return isinstance(text, str) and bool(text) and len(text) <= 128 and (text[0].isascii() and (text[0].isalpha() or text[0] == "_")) and all(c.isascii() and (c.isalnum() or c == "_") for c in text)


def lex(source, budget):
if not isinstance(source, str) or len(source) > 100_000:
raise ReductionError("resource_limit", "Expression must be a string of at most 100000 characters")
budget.charge(len(source))
tokens, pos = [], 0
while pos < len(source):
start, c = pos, source[pos]
if c.isspace():
pos += 1
continue
if c == "{":
depth = 1
pos += 1
while pos < len(source) and depth:
if source[pos] == "{":
depth += 1
if depth > 64:
raise ReductionError("resource_limit", "Comment nesting limit")
elif source[pos] == "}":
depth -= 1
pos += 1
if depth:
raise ReductionError("syntax", "Unclosed comment")
continue
if c == "}":
raise ReductionError("syntax", "Unexpected comment terminator")
if c == '"':
pos += 1
while pos < len(source):
if source[pos] == "\\":
if pos + 1 >= len(source) or source[pos + 1] not in ('"', "\\", "n"):
raise ReductionError("syntax", "Unsupported quoted identifier escape")
pos += 2
elif source[pos] == '"':
pos += 1
break
else:
pos += 1
else:
raise ReductionError("syntax", "Unclosed quoted identifier")
text = source[start:pos]
try:
json.loads(text)
except ValueError as exc:
raise ReductionError("syntax", "Invalid quoted identifier") from exc
tokens.append(Token("quoted", text, start, pos))
continue
if c.isascii() and (c.isalpha() or c == "_"):
pos += 1
while pos < len(source) and source[pos].isascii() and (source[pos].isalnum() or source[pos] == "_"):
pos += 1
tokens.append(Token("ident", source[start:pos], start, pos))
continue
pos += 1
tokens.append(Token("punct", c, start, pos))
return tokens


def reference(tokens, budget):
"""Parse only a reference, never evaluate arithmetic or source strings."""
if len(tokens) == 1 and tokens[0].kind == "quoted":
# Whole-reference quoting is supported as an additive spelling. A quoted
# function-like name remains an identifier and is never scanned as code.
decoded = json.loads(tokens[0].text)
if any(c in decoded for c in ('{', '}', '"', '\\')):
return None
inner = lex(decoded, budget)
if any(t.kind == "quoted" for t in inner):
return None
return reference(inner, budget)
if not tokens or tokens[0].kind != "ident" or not ident(tokens[0].text):
return None
parts, pos = [tokens[0].text], 1
while pos + 1 < len(tokens) and tokens[pos].text == "." and tokens[pos + 1].kind == "ident":
if not ident(tokens[pos + 1].text):
return None
parts.append(tokens[pos + 1].text)
pos += 2
if pos == len(tokens):
return Reference(".".join(parts))
if tokens[pos].text != "[" or tokens[-1].text != "]":
return None
tail = tokens[pos + 1:-1]
if not tail or len(tail) % 2 == 0:
return None
if any(t.kind != "ident" or not ident(t.text) for t in tail[::2]) or any(t.text != "," for t in tail[1::2]):
return None
return Reference(".".join(parts), tuple(t.text for t in tail[::2]))


def lower_expression(source, scope, budget=None):
"""Return retained source + scalar text + typed reduction/source spans.

Caller must share Budget across the entire expanded model. This pass neither
resolves ordinary scalar references nor validates scalar syntax/functions.
"""
budget = budget if budget is not None else Budget()
tokens = lex(source, budget)
stack, calls = [], []
# Balanced iterative traversal; no input-dependent Python recursion.
for index, token in enumerate(tokens):
if token.text in ("(", "["):
if len(stack) >= 64:
raise ReductionError("resource_limit", "Delimiter nesting limit")
stack.append((index, token.text, []))
elif token.text == "," and stack and stack[-1][1] == "(":
stack[-1][2].append(index)
elif token.text in (")", "]"):
if not stack or stack[-1][1] != (")" == token.text and "(" or "["):
raise ReductionError("syntax", "Mismatched delimiter")
opening, kind, commas = stack.pop()
if kind == "(" and opening and tokens[opening - 1].kind == "ident" and (opening < 2 or tokens[opening - 2].text != "."):
calls.append((opening - 1, opening, index, commas))
if stack:
raise ReductionError("syntax", "Unclosed delimiter")
replacements, nodes = [], []
for fn, opening, closing, commas in calls:
budget.charge(1)
op = tokens[fn].text.upper()
if op not in ("SUM", "MEAN", "MIN", "MAX"):
continue
starts, ends = [opening + 1] + [c + 1 for c in commas], commas + [closing]
budget.charge(closing - opening)
args = [tokens[a:b] for a, b in zip(starts, ends)]
refs = [reference(arg, budget) for arg in args]
if len(args) != 1 or refs[0] is None:
if op in ("SUM", "MEAN"):
raise ReductionError("unsupported_semantics", "Reduction requires exactly one declared reference")
continue # Existing scalar MIN/MAX syntax is the scalar parser's job.
ref = refs[0]
try:
selection = scope.select(ref, budget)
except ReductionError as exc:
if op in ("MIN", "MAX") and ref.selectors is None and exc.code == "unknown_reference":
continue # Preserve scalar unknown-reference/function error behavior.
raise
if op in ("MIN", "MAX") and not selection.reduced_axes:
continue # A literal cell/scalar does not become a one-element array.
names = [json.dumps(n) for n in selection.names]
if not names:
raise ReductionError("empty_reduction", "Empty reductions have no implicit identity")
if op in ("SUM", "MEAN"):
text = "(" + " + ".join(names) + ")"
if op == "MEAN":
text = "(" + text + " / " + str(len(names)) + ")"
else:
# Singleton MIN/MAX is identity; Python min(number) is not.
text = names[0] if len(names) == 1 else op + "(" + ", ".join(names) + ")"
start, end = tokens[fn].start, tokens[closing].end
budget.charge(len(replacements))
if any(start < b and a < end for a, b, _ in replacements):
raise ReductionError("unsupported_semantics", "Nested reduction expressions are unsupported")
budget.charge(len(text.encode("utf-8")))
replacements.append((start, end, text))
nodes.append(Reduction(op, ref, selection, start, end))
pieces, pos = [], 0
for start, end, text in sorted(replacements):
pieces.extend((source[pos:start], text))
pos = end
pieces.append(source[pos:])
equation = "".join(pieces)
if len(equation) > 100_000:
raise ReductionError("resource_limit", "Expanded expression length limit")
return Lowered(source, equation, tuple(sorted(nodes, key=lambda n: n.start)), tuple(sorted(replacements)))
68 changes: 42 additions & 26 deletions app/core/structured.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import json
import re
from app.core.equations import canonical_name
from app.core.array_reductions import Budget, Selection, ReductionError, lower_expression

MAX_CELLS = 4096
MAX_MODULE_DEPTH = 8
Expand Down Expand Up @@ -215,36 +216,36 @@ def resolve(scope, reference, context, aggregate=False, trail=()):
fail("invalid_index", f"Unknown {dimension} member {selector}")
return [symbol["cells"][idx] for idx in product(*choices)]

expression_work = 0
reduction_budget = Budget()
def equation(scope, expr, context):
nonlocal expression_work
if not isinstance(expr, str):
fail("invalid_equation", "Equations must be strings")
expression_work += len(expr)
if expression_work > 2_000_000 or len(expr) > 100_000:
fail("resource_limit", "Structured equation expansion budget exceeded")
# SUM deliberately accepts one reference only; arbitrary reduction expressions are rejected.
sums = {}
def sum_ref(m):
nonlocal expression_work
values = resolve(scope, m.group(1), context, True)
expression_work += sum(len(v["name"]) + 5 for v in values)
if expression_work > 2_000_000:
fail("resource_limit", "Structured equation expansion budget exceeded")
marker = f"__structured_sum_{len(sums)}"
while marker in expr or marker in scope["symbols"] or marker in scope["bindings"]:
marker += "_"
sums[marker] = "(" + " + ".join(json.dumps(v["name"]) for v in values) + ")"
return marker
reduction_pattern = r'"(?:[^"\\]|\\.)*"|\bSUM\s*\(\s*([A-Za-z_][A-Za-z0-9_.]*(?:\[[^\[\]]*\])?)\s*\)'
expr = re.sub(reduction_pattern, lambda m: m.group() if m.group().startswith('"') else sum_ref(m), expr, flags=re.I)
unquoted = re.sub(r'"(?:[^"\\]|\\.)*"', "", expr)
if re.search(r"\bSUM\s*\(", unquoted, re.I):
fail("unsupported_semantics", "SUM supports one array reference only")
class ValidatedSelection:
def select(self, ref, budget):
# Ports have already been validated as explicit scalar references.
# Only local declared shapes can introduce reduction axes.
symbol = scope["symbols"].get(ref.name)
axes = symbol["shape"] if symbol else ()
selectors = axes if ref.selectors is None else ref.selectors
reduced = tuple(d for d, selector in zip(axes, selectors) if d == selector)
reference = ref.name
if ref.selectors is not None:
reference += "[" + ",".join(ref.selectors) + "]"
try:
values = resolve(scope, reference, context, True)
except StructuredModelError as exc:
if exc.code == "unknown_reference":
raise ReductionError(exc.code, str(exc)) from exc
raise
names = tuple(v["name"] for v in values)
budget.charge(sum(len(n.encode("utf-8")) + 5 for n in names))
return Selection(names, reduced)
try:
lowered = lower_expression(expr, ValidatedSelection(), reduction_budget)
except ReductionError as exc:
fail(exc.code, str(exc), scope["prefix"])
def replace(m):
token = m.group()
if token in sums:
return sums[token]
if token.startswith('"'):
values = resolve(scope, json.loads(token), context)
return json.dumps(values[0]["name"])
Expand All @@ -259,7 +260,22 @@ def replace(m):
if re.fullmatch(r"[eE][0-9]+", token) and m.start() and expr[m.start()-1].isdigit():
return token
fail("unknown_reference", f"Unknown reference {token}", scope["prefix"])
return TOKEN.sub(replace, expr)
source = expr
pieces, pos = [], 0
for start, end, text in lowered.replacements:
expr = source[pos:start]
pieces.extend((TOKEN.sub(replace, expr), text))
pos = end
expr = source[pos:]
pieces.append(TOKEN.sub(replace, expr))
equation = "".join(pieces)
if len(equation) > 100_000:
fail("resource_limit", "Expanded expression length limit")
try:
reduction_budget.charge(len(equation))
except ReductionError as exc:
fail(exc.code, str(exc), scope["prefix"])
return equation

for scope in scopes.values():
for name, symbol in scope["symbols"].items():
Expand Down
24 changes: 24 additions & 0 deletions docs/ARRAY-REDUCTION-ADAPTER-REVIEW.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Independent typed array adapter review

2026-10-24. Reviewer legacy_repair_resume; independent of adapter author. Approve the exact four-file implementation/test/reference package below for scoped local commit and subsequent public review. This is not release authorization or a scientific-validity certificate. No runtime files were edited by this reviewer.

Independently ran adapter, structured-model, structured-integration and offline research suites: **143 passed, one pre-existing Pydantic incomplete-forward-reference warning, 1.53 s**. An additional 18 independent checks covered nonalphabetic declared two-axis order, cancellation-sensitive left-associated SUM/MEAN, ordered literal subdomains, whole-reference quoting, scalar outer MIN/MAX, unchanged unary scalar/literal-cell forms, repeated reductions without marker text, input dictionary preservation, arithmetic/list/nested unsupported reductions, quoted comment injection and nesting caps. All passed. These scratch checks were executed independently; they are not claimed as an additional committed regression suite.

The adapter reuses the validated compiler resolver and global dimensions/module checks. A shared Budget covers every expanded equation/override; generated scalar text is charged and bounded, while existing 4096-cell, 128-instance, module-depth and identifier budgets remain. Typed replacement spans are substituted between separately rewritten scalar source segments, with no user-collidable placeholder. Declared rightmost-axis-fastest order is retained; no sorted enumeration or zero-prefix SUM reassociation is introduced. Scalar multiargument MIN/MAX and unary scalar/literal-cell calls retain existing validation paths. Unary reductions require a declared selected axis, including singleton axes. Module input/output validation still requires explicit scalar targets and forbids parent capture; the reducer does not infer array shape from dotted spelling.

Original project dictionaries, stock initializers, override selection, source-map entries and flow endpoint resolution are untouched by the adapter. Existing API integration regressions exercise original structured JSON save/reload and compile/run; typed reductions are not an XMILE interoperability implementation. Pinned executed reference receipt contains all 33 samples for the four reductions; adapter tests compare these numerical arrays. This is narrow compatibility evidence, not equivalence to the separate inspected Simlin source revision, complete language coverage, performance evidence or empirical model verification.

No arbitrary array expression operands, slices, wildcards, mappings, array ports, composite nested reductions or scalar MEAN lists are admitted. No parser limits are relaxed, generated authored code executed, model definitions changed or fake equation provenance introduced. Required release work remains the full suite result, reviewed documentation/public allowlist, API/MCP/browser checks appropriate to the final release and exact CI/staged deployment gates; no push/deploy performed by this reviewer.

## Exact reviewed bytes

- app/core/structured.py: 44e119463fc6470e867fcfd519099f0ebb2a527a547c88d623ae2b1f7591a5ee
- app/core/array_reductions.py: 9881875542167f033142376167e3e13487000aaed84029d157a09e200f2c275b
- tests/test_array_reductions.py: 0bb1d51f6ba256bcf828e4e383bc15c9229fa58ef1a8c9b99c9b4bc8503f61f2
- docs/array-reduction-simlin-reference.json: 1372e7c9e11317fad3f80b46632923980a3c67f245f6006345e4d9405a09520d

Changing those bytes requires renewed exact-byte review. Research contract and primary-source/rights limitations remain in docs/ARRAY-REDUCTION-RESEARCH.md; frozen research artifact hashes are recorded in docs/ARRAY-REDUCTION-ADAPTER.md; this review does not claim fresh upstream source retrieval.

## Independent next-release default-audit receipt review

Approved receipt-only refresh for a future release. Independently verified full audit SHA256 204e0b30913ea1c881545a7539c84e5258d909f2c9e1af226ed2e194e473153e and public index SHA256 338942c218df43b74f47d8a3b90c14cfef2b07d77d4171c39f0106bce881028a. All 993 full-audit rows report passed, match the exact Git-tracked model path set and current on-disk SHA256, and extract exactly to the public ordered id/hash/status rows. Every engine hash matches its current file and the full core module set. Compared with committed HEAD public index, all fields and ordered model rows are identical except the updated structured.py engine hash and added array_reductions.py hash. The evidence records 90.73 seconds and fallback seed42; reviewer checked extraction and hash bindings rather than rerunning that 90-second audit. Finite defaults remain numerical regression evidence, not source/scientific validation. This is a next-release receipt; no current public worktree, PR8 or deployment was changed by this reviewer.
Loading
Loading