diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8267b..dbef234 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project are documented here. ## [Unreleased] +### Repository + +- Repository validation passes again. Thirteen recorded measurement files that already contain local paths are exempt from the local-path check only while their pinned bytes are unchanged, a negated sentence no longer trips the public-claim check, and a linked worktree's `.git` file is no longer scanned. + ### Research and documentation - Added an independently preflighted SymPy ArraySymbol development pair. Both arms passed the frozen 35 external and 48 selected upstream checks; ON used fewer provider-reported tokens and took longer. A post-hoc check found an overbroad `_ArrayExpr` change and the same `ZeroArray`/`OneArray` regression in both arms, invalidating an accepted-repair economy claim. Added two primary studies on agent instruction adherence to the cost-frontier review. diff --git a/docs/research/2026-09-25-handoff-debt.md b/docs/research/2026-09-25-handoff-debt.md index a7f8a89..23fa5af 100644 --- a/docs/research/2026-09-25-handoff-debt.md +++ b/docs/research/2026-09-25-handoff-debt.md @@ -15,4 +15,4 @@ The authors' [reference code](https://github.com/anjilab/agent-handoff-debt) inc - Count diagnosis, handoff generation, all continuation requests including retries and late completions, cached and uncached input, output, and elapsed time. A shared diagnosis is counted in each potential deployment policy total; actual experimental spend is reported separately. Evaluate both complete source checkpoints externally and blind to arm. - Keep repetitions within one repository lineage clustered. The [old exposed packaging pair](../measurements/2026-09-24-phase-reset-development.md) saved 20.0% in CLI-reported input-plus-output but increased uncached input and time, and lacked complete provider accounting. The [synthetic transport probe](../measurements/2026-09-25-phase-reset-transport-exploratory.md) fixed the session-isolation design only. Neither supports an economy or quality claim. -An Astra design review of the proposed 2–3-lineage development A/B identified five launch gates: fixed diagnosis/handoff policy, equal branch state and isolation, balanced order with lineage-level analysis, frozen complete provider accounting, and independently qualified external quality checks. No real-model phase-reset A/B has passed these gates. The review recommended prioritizing the simpler native quiet-diagnostic hypothesis for the next development screen; its [previous four-run screen](../measurements/2026-09-25-quiet-diagnostic-model-screen-development.md) failed quality and first-command audit gates, so a new screen also requires corrected measurement qualification and new prospective tasks. This is an experiment-order decision, not evidence that quiet diagnostics save tokens or preserve quality. +An Astra design review of the proposed 2–3-lineage development A/B identified five launch gates: fixed diagnosis/handoff policy, equal branch state and isolation, balanced order with lineage-level analysis, frozen complete provider accounting, and independently qualified external quality checks. No real-model phase-reset A/B has passed these gates. The review recommended prioritizing the simpler native quiet-diagnostic hypothesis for the next development screen; its [previous four-run screen](../measurements/2026-09-25-quiet-diagnostic-model-screen-development.md) failed quality and first-command audit gates, so a new screen also requires corrected measurement qualification and new prospective tasks. This is an experiment-order decision, not evidence that quiet diagnostics lower token use or preserve quality. diff --git a/scripts/test_release_tools.py b/scripts/test_release_tools.py index f0dc3f9..632dd99 100644 --- a/scripts/test_release_tools.py +++ b/scripts/test_release_tools.py @@ -419,6 +419,31 @@ def test_public_readmes_cover_installation_limits_and_attribution(self) -> None: self.assertIn(literal, text, f"{relative} missing {literal!r}") self.assertIn(model_disclaimer, text) + def test_frozen_evidence_keeps_local_paths_only_while_its_bytes_match(self) -> None: + relative = "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-positive-trace.jsonl" + source = (ROOT / relative).read_bytes() + repo = self.fixture_repo() + target = repo / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(source) + result = self.run_validator(repo) + self.assertEqual(result.returncode, 0, result.stderr) + + target.write_bytes(source.replace(b"\n", b"\r\n")) + result = self.run_validator(repo) + self.assertEqual(result.returncode, 0, result.stderr) + + target.write_bytes(source + b"\n") + result = self.run_validator(repo) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"{relative}: local absolute path", result.stderr) + + def test_worktree_git_file_is_not_scanned(self) -> None: + repo = self.fixture_repo() + (repo / ".git").write_text("gitdir: " + "/" + "Users/example/repo/.git/worktrees/fix\n", encoding="utf-8") + result = self.run_validator(repo) + self.assertEqual(result.returncode, 0, result.stderr) + def test_validator_rejects_unsupported_public_claims(self) -> None: repo = self.fixture_repo() (repo / "README.md").write_text( diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index 41c3f35..6e4a685 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import re import stat @@ -45,6 +46,38 @@ if parent.as_posix() != "." ) LOCAL_PATH_RE = re.compile(r"/(?:Users|home)/[^/\s]+/") +# Recorded measurement evidence that already contains local paths. Most files +# are pinned by SHA-256 from other tracked results and rechecked by reducers, +# so rewriting them would break the evidence chain. The exemption covers only +# the local-path check and lapses as soon as a file's bytes change. +FROZEN_LOCAL_PATH_EVIDENCE = { + "docs/measurements/data/2026-09-25-click-source-only-v3-normal-cli-exploratory-trace.jsonl": + "fd75a0132616931730be0ba573bf1adf9d67ad8634759696418df353ce98dc4e", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-negative-trace.jsonl": + "108c164043a3467aa80ec4139054e68ad60a6bb58400ddb79b27c4f03935f8a0", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-positive-path-trace.jsonl": + "3a69e86077450b1e24396d599d205a166188e31e73b45b41f1afed8995156ff1", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-integrated-wrong_cleanup_uncertain-trace.jsonl": + "ff5d25c140b5300389ae0d7c5385ad1491c8e2696b54fe3d428d2a08c3bc8fe6", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-integrated-wrong_complete-trace.jsonl": + "fcf7a11064aaa2e2bc0ea8ebd61864377b1c6cbac21ae6c8fea59b464ff03630", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-negative-trace.jsonl": + "9592f468e648ef5c1debedb8ffd0ead8965a1fe916dfd40e7142d27b1bc3f639", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-01-b1-packaging-verbose-trace.jsonl": + "d2624a3c2164b392207a00819ae46bf1caa8bf49468ca962917488c108ed7191", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-02-b1-packaging-quiet-trace.jsonl": + "031771f6c5c6f7dd8bd2aa393161309ac96ce50ca195cf079667bad9ebaad9bd", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-03-b1-click-quiet-trace.jsonl": + "39558704ea89bf61ebca93586d05486030788bd620e43b26dcf72ba064d5eb84", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-normal-cli-04-b1-click-verbose-trace.jsonl": + "da03be3444bc4bee62cb9848d6e4e59d308833c40e72bbde865cf20729eca60b", + "docs/measurements/data/2026-09-25-quiet-variance-bundle-v3-positive-trace.jsonl": + "cfec1cf9a0394c2a656e75676e9862602601a795a8457820fcaa6c74ced5009d", + "docs/measurements/data/2026-09-25-quiet-variance-calibration-bundle-v4-candidate.json": + "e3f3203c07a199e47a3ccc261d4317352bacb703dd2a3288fddeab7826291f68", + "docs/research/data/2026-09-25-quiet-variance-bundle-v3-integrated-expectations.json": + "878f41c2521aa4551593cfc8797b8b78c1fb0c8ef7d1af31fbfcb329a1bb481e", +} CREDENTIAL_RE = re.compile(r"\bsk-[A-Za-z0-9_-]{12,}\b") KNOWN_SYNTHETIC_TOKENS = {"sk-abcdefghijklmnopqrstuvwxyz123456"} PRIVATE_KEY_RE = re.compile( @@ -74,7 +107,8 @@ def public_relative_paths(repo: Path) -> Iterable[Path]: continue relative = path.relative_to(repo) rendered = relative.as_posix() - if rendered in {".DS_Store"} or rendered.startswith(IGNORED_PREFIXES): + # In a linked worktree `.git` is a file that names the main checkout. + if rendered in {".DS_Store", ".git"} or rendered.startswith(IGNORED_PREFIXES): continue yield relative @@ -187,9 +221,9 @@ def prohibited_path_reason(relative: Path) -> str | None: return None -def scan_text(label: str, text: str) -> list[str]: +def scan_text(label: str, text: str, allow_local_paths: bool = False) -> list[str]: errors: list[str] = [] - if LOCAL_PATH_RE.search(text): + if not allow_local_paths and LOCAL_PATH_RE.search(text): errors.append(f"{label}: local absolute path") if any(match.group(0) not in KNOWN_SYNTHETIC_TOKENS for match in CREDENTIAL_RE.finditer(text)): errors.append(f"{label}: credential marker") @@ -217,7 +251,10 @@ def validate_tracked_files(repo: Path) -> list[str]: text = content.decode("utf-8") except UnicodeDecodeError: continue - errors.extend(scan_text(relative.as_posix(), text)) + # Windows checkouts may convert line endings; the digest covers LF bytes. + normalized = content.replace(b"\r\n", b"\n") + frozen = FROZEN_LOCAL_PATH_EVIDENCE.get(relative.as_posix()) == hashlib.sha256(normalized).hexdigest() + errors.extend(scan_text(relative.as_posix(), text, allow_local_paths=frozen)) return errors