Retain check results and carry repository notes

bin/jpdev

@@ -82,7 +82,9 @@ cmd_check() {
   export JPDEV_LOGDIR=$logdir
   "$script" --manifest "$manifest" "${passthru[@]}"
   local status=$?
-  jpdev_log "full output of each check: $logdir"
+  if compgen -G "$logdir/*.log" > /dev/null; then
+    jpdev_log "full output of each check: $logdir"
+  fi
   exit "$status"
 }
 
@@ -475,7 +477,7 @@ codex_review() {
 
 # codex_check_brief [--build] FILE: check the build heading and the filter-list words in FILE.
 codex_check_brief() {
-  local file="" build=0
+  local file="" build=0 root manifest
   while [ $# -gt 0 ]; do
     case $1 in
       --build) build=1; shift ;;
@@ -492,6 +494,25 @@ codex_check_brief() {
     fi
     echo "jpdev codex check-brief: warning: $file has no line exactly '## Before you write msg.txt'" >&2
   fi
+  if [ "$build" -eq 1 ]; then
+    root=$(git rev-parse --show-toplevel 2>/dev/null || true)
+    manifest=${root:+$root/dev.json}
+    if [ -n "$manifest" ] && [ -f "$manifest" ] && python3 -I - "$manifest" <<'PY'
+import json, sys
+try:
+    notes = json.load(open(sys.argv[1], encoding='utf-8')).get('notes', [])
+except (OSError, ValueError, AttributeError):
+    notes = []
+if not isinstance(notes, list): notes = []
+sys.exit(0 if notes else 1)
+PY
+    then
+      if ! grep -qxF "## This repository's notes" "$file"; then
+        echo "jpdev codex check-brief: refused: $file has no line exactly '## This repository's notes', required by $manifest" >&2
+        exit 1
+      fi
+    fi
+  fi
   local out status
   out=$(python3 -I "$kit/lib/codex_words.py" "$file" "$kit/config/codex-words.txt" "$JPDEV_CODEX_WORDS")
   status=$?
@@ -588,7 +609,7 @@ codex_run() {
     done
   fi
   brief=$(cd "$(dirname "$brief")" && pwd)/$(basename "$brief")
-  codex_check_brief --build "$brief"
+  (cd "$dir" && codex_check_brief --build "$brief") || exit $?
   if [ -z "$log" ]; then
     mkdir -p "$JPDEV_SCRATCH/logs"
     log=$JPDEV_SCRATCH/logs/codex-$(basename "$dir")-$(date +%Y%m%dT%H%M%S).log
@@ -837,7 +858,28 @@ codex_publish() {
 # codex_brief_template prints the brief skeleton a Codex build or review starts from.
 codex_brief_template() {
   case ${1:-} in
-    "") cat "$kit/templates/codex-brief.md" ;;
+    "")
+      cat "$kit/templates/codex-brief.md"
+      local root manifest notes
+      root=$(git rev-parse --show-toplevel 2>/dev/null || true)
+      manifest=${root:+$root/dev.json}
+      if [ -n "$manifest" ] && [ -f "$manifest" ]; then
+        notes=$(python3 -I - "$manifest" <<'PY'
+import json, sys
+try:
+    notes = json.load(open(sys.argv[1], encoding='utf-8')).get('notes', [])
+except (OSError, ValueError, AttributeError):
+    notes = []
+if not isinstance(notes, list): notes = []
+for note in notes:
+    if isinstance(note, str): print('- ' + note)
+PY
+)
+        if [ -n "$notes" ]; then
+          printf "\n## This repository's notes\n\n%s\n" "$notes"
+        fi
+      fi
+      ;;
     --review) [ $# -eq 1 ] || jpdev_die "codex brief-template: --review takes no value"; cat "$kit/templates/codex-review-brief.md" ;;
     *) jpdev_die "codex brief-template: unknown argument $1" ;;
   esac

docs/adopting.md

@@ -19,6 +19,12 @@ Start from `templates/dev.json` and keep what the repository runs. The manifest
 - `setup`: steps `jpdev setup` and CI run once after a clone (`npm ci`, the python
   requirements).
 - `checks`: the check list, in the order it should run (cheap checks first).
+- `notes`: optional short sentences about repository-specific traps. For example,
+  `"run every Python command with the venv on the PATH"` and
+  `"a change to a file listed in scripts/parity.sh updates its row"`. They are plain text:
+  `jpdev codex brief-template` prints them under `## This repository's notes`, and
+  `jpdev codex check-brief --build` requires that heading when notes are present; no note is
+  executed.
 - `mutants`: the command `jpdev mutants` runs; `{rows}` becomes one `--row 'TEXT'` per row.
 - `parity`: an optional last check that compares the repository against another one; it names
   the environment variable it needs in `requires_env`, and is skipped, and said to be skipped,
@@ -69,6 +75,12 @@ step that invokes `python3`. `jpdev check` uses the vendored copy when it exists
 `JPDEV_WRAP` so that each step runs under its toolchain's cap. `jpdev env` shows the interpreter
 selected for the current repository's manifest.
 
+`jpdev check` stores each step's complete step output (stdout and stderr combined) at
+`$JPDEV_SCRATCH/logs/<repository>-<timestamp>/<step>.log`. If that output cannot be persisted, the
+summary keeps the completed step's result and reports the persistence failure separately, naming
+the intended log target. Remaining steps are marked not run, and the infrastructure failure does
+not change the check-derived exit status. No log path is advertised unless a complete log exists.
+
 ## 3. Add the agent instructions
 
 Copy `templates/AGENTS.md` to the repository root and generate `CLAUDE.md` from it with

lib/validate_manifest.py

@@ -42,6 +42,14 @@ def main():
                     problems.append("(root): %r is required" % key)
             if manifest.get("version") != 1:
                 problems.append("version: must be 1")
+            if "notes" in manifest:
+                notes = manifest["notes"]
+                if not isinstance(notes, list):
+                    problems.append("notes: not an array")
+                else:
+                    for index, note in enumerate(notes):
+                        if not isinstance(note, str) or not note or len(note) > 240:
+                            problems.append("notes/%d: must be a non-empty string of at most 240 characters" % index)
             for kind in ("setup", "checks"):
                 for index, step in enumerate(manifest.get(kind, [])):
                     if not isinstance(step, dict) or not step.get("name") or not step.get("run"):

scripts/dev/check.sh

@@ -18,6 +18,22 @@ while args:
     elif a in ('--all', '--setup'): opt[a[2:]] = True
     else: sys.exit('check.sh: unknown argument ' + a)
 m = json.load(open(opt['manifest'], encoding='utf-8'))
+logdir = os.environ.get('JPDEV_LOGDIR')
+log_probe_error = ''
+if logdir:
+    probe = os.path.join(logdir, '.jpdev-write-probe-%d' % os.getpid())
+    try:
+        os.makedirs(logdir, exist_ok=True)
+        descriptor = os.open(probe, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        try: os.write(descriptor, b'x')
+        finally: os.close(descriptor)
+        os.unlink(probe)
+    except OSError as error:
+        log_probe_error = error.strerror or str(error)
+        try:
+            if os.path.exists(probe): os.unlink(probe)
+        except OSError:
+            pass
 python = m.get('toolchains', {}).get('python', {})
 venv = python.get('venv')
 run_env = dict(os.environ)
@@ -59,10 +75,11 @@ if venv is not None:
 steps = m.get('setup', []) if opt['setup'] else m['checks'] + ([dict(name='parity', **m['parity'])] if 'parity' in m else [])
 unknown = (opt['only'] | opt['skip']) - set(s['name'] for s in steps)
 if unknown: print('check.sh: no step named ' + ', '.join(sorted(unknown)), file=sys.stderr); sys.exit(2)
-wrap, logdir, rows, failed = os.environ.get('JPDEV_WRAP'), os.environ.get('JPDEV_LOGDIR'), [], 0
+wrap, rows, failed, infrastructure = os.environ.get('JPDEV_WRAP'), [], 0, ''
 for c in steps:
     n = c['name']
     if (opt['only'] and n not in opt['only']) or n in opt['skip']: continue
+    if infrastructure: rows.append((n, 'NOT RUN', '', [])); continue
     if failed and not opt['all']: rows.append((n, 'NOT RUN', '', [])); continue
     if c.get('requires_env') and not os.environ.get(c['requires_env']):
         # A skip is a failure under CI (CI is set) unless the step says optional: true.
@@ -85,18 +102,35 @@ for c in steps:
             found = subprocess.run([executable, '--version'], text=True, stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT, env=step_env).stdout.strip()
             took += ', python %s (%s)' % (executable, found)
-    if logdir: os.makedirs(logdir, exist_ok=True); open('%s/%s.log' % (logdir, n), 'w').write(out)
     hit = [l.strip() for l in lines if c.get('summary') and re.search(c['summary'], l)]
     note = ['    ' + l for l in (hit if len(hit) <= 3 else ['%d lines matched; last: %s' % (len(hit), hit[-1])])]
-    if p.returncode == 0: rows.append((n, 'PASS', took, note)); continue
-    failed += 1
-    rows.append((n, 'FAIL', '%s, exit %d' % (took, p.returncode), note + ['    --'] + ['    ' + l for l in lines[:40]] + (['    ... %d more lines' % (len(lines) - 40)] if len(lines) > 40 else [])))
+    if p.returncode == 0:
+        rows.append((n, 'PASS', took, note))
+    else:
+        failed += 1
+        rows.append((n, 'FAIL', '%s, exit %d' % (took, p.returncode), note + ['    --'] + ['    ' + l for l in lines[:40]] + (['    ... %d more lines' % (len(lines) - 40)] if len(lines) > 40 else [])))
+    if logdir:
+        log = '%s/%s.log' % (logdir, n)
+        pending = log + '.pending-%d' % os.getpid()
+        try:
+            os.makedirs(logdir, exist_ok=True)
+            with open(pending, 'w', encoding='utf-8') as handle: handle.write(out)
+            os.replace(pending, log)
+        except OSError as error:
+            try:
+                if os.path.exists(pending): os.unlink(pending)
+            except OSError:
+                pass
+            infrastructure = 'cannot persist the completed check log %s: %s' % (log, error.strerror or log_probe_error or error)
 print('== %s summary (%s) ==' % ('setup' if opt['setup'] else 'check', os.path.basename(opt['manifest'])))
 if not opt['setup'] and 'handover' in m:
     h = m['handover']
     print('handover: boundary %d paths, shared %d paths; see AGENTS.md "Before you hand over"' %
           (len(h.get('boundary_paths', [])), len(h.get('shared_paths', []))))
+if infrastructure: print('infrastructure failure: ' + infrastructure)
 for n, s, head, extra in rows: print('\n'.join(['%-8s %-24s %s' % (s, n, head)] + extra))
-print('== %d passed, %d failed, %d skipped, %d not run ==' % tuple(sum(1 for r in rows if r[1] == k) for k in ('PASS', 'FAIL', 'SKIP', 'NOT RUN')))
+totals = tuple(sum(1 for r in rows if r[1] == k) for k in ('PASS', 'FAIL', 'SKIP', 'NOT RUN'))
+if infrastructure: print('== %d passed, %d failed, %d skipped, %d not run; infrastructure failure ==' % totals)
+else: print('== %d passed, %d failed, %d skipped, %d not run ==' % totals)
 sys.exit(min(failed, 255))
 PY

templates/check.sh

@@ -18,6 +18,22 @@ while args:
     elif a in ('--all', '--setup'): opt[a[2:]] = True
     else: sys.exit('check.sh: unknown argument ' + a)
 m = json.load(open(opt['manifest'], encoding='utf-8'))
+logdir = os.environ.get('JPDEV_LOGDIR')
+log_probe_error = ''
+if logdir:
+    probe = os.path.join(logdir, '.jpdev-write-probe-%d' % os.getpid())
+    try:
+        os.makedirs(logdir, exist_ok=True)
+        descriptor = os.open(probe, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        try: os.write(descriptor, b'x')
+        finally: os.close(descriptor)
+        os.unlink(probe)
+    except OSError as error:
+        log_probe_error = error.strerror or str(error)
+        try:
+            if os.path.exists(probe): os.unlink(probe)
+        except OSError:
+            pass
 python = m.get('toolchains', {}).get('python', {})
 venv = python.get('venv')
 run_env = dict(os.environ)
@@ -59,10 +75,11 @@ if venv is not None:
 steps = m.get('setup', []) if opt['setup'] else m['checks'] + ([dict(name='parity', **m['parity'])] if 'parity' in m else [])
 unknown = (opt['only'] | opt['skip']) - set(s['name'] for s in steps)
 if unknown: print('check.sh: no step named ' + ', '.join(sorted(unknown)), file=sys.stderr); sys.exit(2)
-wrap, logdir, rows, failed = os.environ.get('JPDEV_WRAP'), os.environ.get('JPDEV_LOGDIR'), [], 0
+wrap, rows, failed, infrastructure = os.environ.get('JPDEV_WRAP'), [], 0, ''
 for c in steps:
     n = c['name']
     if (opt['only'] and n not in opt['only']) or n in opt['skip']: continue
+    if infrastructure: rows.append((n, 'NOT RUN', '', [])); continue
     if failed and not opt['all']: rows.append((n, 'NOT RUN', '', [])); continue
     if c.get('requires_env') and not os.environ.get(c['requires_env']):
         # A skip is a failure under CI (CI is set) unless the step says optional: true.
@@ -85,18 +102,35 @@ for c in steps:
             found = subprocess.run([executable, '--version'], text=True, stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT, env=step_env).stdout.strip()
             took += ', python %s (%s)' % (executable, found)
-    if logdir: os.makedirs(logdir, exist_ok=True); open('%s/%s.log' % (logdir, n), 'w').write(out)
     hit = [l.strip() for l in lines if c.get('summary') and re.search(c['summary'], l)]
     note = ['    ' + l for l in (hit if len(hit) <= 3 else ['%d lines matched; last: %s' % (len(hit), hit[-1])])]
-    if p.returncode == 0: rows.append((n, 'PASS', took, note)); continue
-    failed += 1
-    rows.append((n, 'FAIL', '%s, exit %d' % (took, p.returncode), note + ['    --'] + ['    ' + l for l in lines[:40]] + (['    ... %d more lines' % (len(lines) - 40)] if len(lines) > 40 else [])))
+    if p.returncode == 0:
+        rows.append((n, 'PASS', took, note))
+    else:
+        failed += 1
+        rows.append((n, 'FAIL', '%s, exit %d' % (took, p.returncode), note + ['    --'] + ['    ' + l for l in lines[:40]] + (['    ... %d more lines' % (len(lines) - 40)] if len(lines) > 40 else [])))
+    if logdir:
+        log = '%s/%s.log' % (logdir, n)
+        pending = log + '.pending-%d' % os.getpid()
+        try:
+            os.makedirs(logdir, exist_ok=True)
+            with open(pending, 'w', encoding='utf-8') as handle: handle.write(out)
+            os.replace(pending, log)
+        except OSError as error:
+            try:
+                if os.path.exists(pending): os.unlink(pending)
+            except OSError:
+                pass
+            infrastructure = 'cannot persist the completed check log %s: %s' % (log, error.strerror or log_probe_error or error)
 print('== %s summary (%s) ==' % ('setup' if opt['setup'] else 'check', os.path.basename(opt['manifest'])))
 if not opt['setup'] and 'handover' in m:
     h = m['handover']
     print('handover: boundary %d paths, shared %d paths; see AGENTS.md "Before you hand over"' %
           (len(h.get('boundary_paths', [])), len(h.get('shared_paths', []))))
+if infrastructure: print('infrastructure failure: ' + infrastructure)
 for n, s, head, extra in rows: print('\n'.join(['%-8s %-24s %s' % (s, n, head)] + extra))
-print('== %d passed, %d failed, %d skipped, %d not run ==' % tuple(sum(1 for r in rows if r[1] == k) for k in ('PASS', 'FAIL', 'SKIP', 'NOT RUN')))
+totals = tuple(sum(1 for r in rows if r[1] == k) for k in ('PASS', 'FAIL', 'SKIP', 'NOT RUN'))
+if infrastructure: print('== %d passed, %d failed, %d skipped, %d not run; infrastructure failure ==' % totals)
+else: print('== %d passed, %d failed, %d skipped, %d not run ==' % totals)
 sys.exit(min(failed, 255))
 PY

templates/dev.schema.json

@@ -14,6 +14,15 @@
     "version": {
       "const": 1
     },
+    "notes": {
+      "type": "array",
+      "description": "short repository-specific sentences printed in Codex build briefs as plain text",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 240
+      }
+    },
     "handover": {
       "type": "object",
       "additionalProperties": false,