Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ All notable changes to this project are documented here.

## [Unreleased]

### Repository

- Repository validation passes again. Thirteen recorded measurement files that already contain local paths are exempt from the local-path check only while their pinned bytes are unchanged, a negated sentence no longer trips the public-claim check, and a linked worktree's `.git` file is no longer scanned.

### Research and documentation

- Added an independently preflighted SymPy ArraySymbol development pair. Both arms passed the frozen 35 external and 48 selected upstream checks; ON used fewer provider-reported tokens and took longer. A post-hoc check found an overbroad `_ArrayExpr` change and the same `ZeroArray`/`OneArray` regression in both arms, invalidating an accepted-repair economy claim. Added two primary studies on agent instruction adherence to the cost-frontier review.
Expand Down
2 changes: 1 addition & 1 deletion docs/research/2026-09-25-handoff-debt.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ The authors' [reference code](https://github.com/anjilab/agent-handoff-debt) inc
- Count diagnosis, handoff generation, all continuation requests including retries and late completions, cached and uncached input, output, and elapsed time. A shared diagnosis is counted in each potential deployment policy total; actual experimental spend is reported separately. Evaluate both complete source checkpoints externally and blind to arm.
- Keep repetitions within one repository lineage clustered. The [old exposed packaging pair](../measurements/2026-09-24-phase-reset-development.md) saved 20.0% in CLI-reported input-plus-output but increased uncached input and time, and lacked complete provider accounting. The [synthetic transport probe](../measurements/2026-09-25-phase-reset-transport-exploratory.md) fixed the session-isolation design only. Neither supports an economy or quality claim.

An Astra design review of the proposed 2–3-lineage development A/B identified five launch gates: fixed diagnosis/handoff policy, equal branch state and isolation, balanced order with lineage-level analysis, frozen complete provider accounting, and independently qualified external quality checks. No real-model phase-reset A/B has passed these gates. The review recommended prioritizing the simpler native quiet-diagnostic hypothesis for the next development screen; its [previous four-run screen](../measurements/2026-09-25-quiet-diagnostic-model-screen-development.md) failed quality and first-command audit gates, so a new screen also requires corrected measurement qualification and new prospective tasks. This is an experiment-order decision, not evidence that quiet diagnostics save tokens or preserve quality.
An Astra design review of the proposed 2–3-lineage development A/B identified five launch gates: fixed diagnosis/handoff policy, equal branch state and isolation, balanced order with lineage-level analysis, frozen complete provider accounting, and independently qualified external quality checks. No real-model phase-reset A/B has passed these gates. The review recommended prioritizing the simpler native quiet-diagnostic hypothesis for the next development screen; its [previous four-run screen](../measurements/2026-09-25-quiet-diagnostic-model-screen-development.md) failed quality and first-command audit gates, so a new screen also requires corrected measurement qualification and new prospective tasks. This is an experiment-order decision, not evidence that quiet diagnostics lower token use or preserve quality.
25 changes: 25 additions & 0 deletions scripts/test_release_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,31 @@ def test_public_readmes_cover_installation_limits_and_attribution(self) -> None:
self.assertIn(literal, text, f"{relative} missing {literal!r}")
self.assertIn(model_disclaimer, text)

def test_frozen_evidence_keeps_local_paths_only_while_its_bytes_match(self) -> None:
relative = "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-positive-trace.jsonl"
source = (ROOT / relative).read_bytes()
repo = self.fixture_repo()
target = repo / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(source)
result = self.run_validator(repo)
self.assertEqual(result.returncode, 0, result.stderr)

target.write_bytes(source.replace(b"\n", b"\r\n"))
result = self.run_validator(repo)
self.assertEqual(result.returncode, 0, result.stderr)

target.write_bytes(source + b"\n")
result = self.run_validator(repo)
self.assertNotEqual(result.returncode, 0)
self.assertIn(f"{relative}: local absolute path", result.stderr)

def test_worktree_git_file_is_not_scanned(self) -> None:
repo = self.fixture_repo()
(repo / ".git").write_text("gitdir: " + "/" + "Users/example/repo/.git/worktrees/fix\n", encoding="utf-8")
result = self.run_validator(repo)
self.assertEqual(result.returncode, 0, result.stderr)

def test_validator_rejects_unsupported_public_claims(self) -> None:
repo = self.fixture_repo()
(repo / "README.md").write_text(
Expand Down
45 changes: 41 additions & 4 deletions scripts/validate_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

import argparse
import hashlib
import json
import re
import stat
Expand Down Expand Up @@ -45,6 +46,38 @@
if parent.as_posix() != "."
)
LOCAL_PATH_RE = re.compile(r"/(?:Users|home)/[^/\s]+/")
# Recorded measurement evidence that already contains local paths. Most files
# are pinned by SHA-256 from other tracked results and rechecked by reducers,
# so rewriting them would break the evidence chain. The exemption covers only
# the local-path check and lapses as soon as a file's bytes change.
FROZEN_LOCAL_PATH_EVIDENCE = {
"docs/measurements/data/2026-09-25-click-source-only-v3-normal-cli-exploratory-trace.jsonl":
"fd75a0132616931730be0ba573bf1adf9d67ad8634759696418df353ce98dc4e",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-negative-trace.jsonl":
"108c164043a3467aa80ec4139054e68ad60a6bb58400ddb79b27c4f03935f8a0",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-positive-path-trace.jsonl":
"3a69e86077450b1e24396d599d205a166188e31e73b45b41f1afed8995156ff1",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-integrated-wrong_cleanup_uncertain-trace.jsonl":
"ff5d25c140b5300389ae0d7c5385ad1491c8e2696b54fe3d428d2a08c3bc8fe6",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-integrated-wrong_complete-trace.jsonl":
"fcf7a11064aaa2e2bc0ea8ebd61864377b1c6cbac21ae6c8fea59b464ff03630",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-negative-trace.jsonl":
"9592f468e648ef5c1debedb8ffd0ead8965a1fe916dfd40e7142d27b1bc3f639",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-01-b1-packaging-verbose-trace.jsonl":
"d2624a3c2164b392207a00819ae46bf1caa8bf49468ca962917488c108ed7191",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-02-b1-packaging-quiet-trace.jsonl":
"031771f6c5c6f7dd8bd2aa393161309ac96ce50ca195cf079667bad9ebaad9bd",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-03-b1-click-quiet-trace.jsonl":
"39558704ea89bf61ebca93586d05486030788bd620e43b26dcf72ba064d5eb84",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-04-b1-click-verbose-trace.jsonl":
"da03be3444bc4bee62cb9848d6e4e59d308833c40e72bbde865cf20729eca60b",
"docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-positive-trace.jsonl":
"cfec1cf9a0394c2a656e75676e9862602601a795a8457820fcaa6c74ced5009d",
"docs/measurements/data/2026-09-25-quiet-variance-calibration-bundle-v4-candidate.json":
"e3f3203c07a199e47a3ccc261d4317352bacb703dd2a3288fddeab7826291f68",
"docs/research/data/2026-09-25-quiet-variance-bundle-v3-integrated-expectations.json":
"878f41c2521aa4551593cfc8797b8b78c1fb0c8ef7d1af31fbfcb329a1bb481e",
}
CREDENTIAL_RE = re.compile(r"\bsk-[A-Za-z0-9_-]{12,}\b")
KNOWN_SYNTHETIC_TOKENS = {"sk-abcdefghijklmnopqrstuvwxyz123456"}
PRIVATE_KEY_RE = re.compile(
Expand Down Expand Up @@ -74,7 +107,8 @@ def public_relative_paths(repo: Path) -> Iterable[Path]:
continue
relative = path.relative_to(repo)
rendered = relative.as_posix()
if rendered in {".DS_Store"} or rendered.startswith(IGNORED_PREFIXES):
# In a linked worktree `.git` is a file that names the main checkout.
if rendered in {".DS_Store", ".git"} or rendered.startswith(IGNORED_PREFIXES):
continue
yield relative

Expand Down Expand Up @@ -187,9 +221,9 @@ def prohibited_path_reason(relative: Path) -> str | None:
return None


def scan_text(label: str, text: str) -> list[str]:
def scan_text(label: str, text: str, allow_local_paths: bool = False) -> list[str]:
errors: list[str] = []
if LOCAL_PATH_RE.search(text):
if not allow_local_paths and LOCAL_PATH_RE.search(text):
errors.append(f"{label}: local absolute path")
if any(match.group(0) not in KNOWN_SYNTHETIC_TOKENS for match in CREDENTIAL_RE.finditer(text)):
errors.append(f"{label}: credential marker")
Expand Down Expand Up @@ -217,7 +251,10 @@ def validate_tracked_files(repo: Path) -> list[str]:
text = content.decode("utf-8")
except UnicodeDecodeError:
continue
errors.extend(scan_text(relative.as_posix(), text))
# Windows checkouts may convert line endings; the digest covers LF bytes.
normalized = content.replace(b"\r\n", b"\n")
frozen = FROZEN_LOCAL_PATH_EVIDENCE.get(relative.as_posix()) == hashlib.sha256(normalized).hexdigest()
errors.extend(scan_text(relative.as_posix(), text, allow_local_paths=frozen))
return errors


Expand Down
Loading