From 387c93dcea6ba2ab7a10c84c17c0c7a495aa9571 Mon Sep 17 00:00:00 2001 From: kikashy Date: Fri, 23 Oct 2026 07:54:53 -0400 Subject: [PATCH 1/4] Add session disposal seals and guarded cleanup Add explicit-root dry-run cleanup for session-created worktrees, reviewer rooms, scratch, shared caches and transcripts. Require disposal inventories, publication and PR evidence, reserves and complete host process visibility, and recheck filesystem and Git state during descriptor-relative removal. Confine builder TMPDIR beneath scratch, exclude nested generated state from mutation copies, and isolate fixture Git discovery. Document cleanup ownership, milestones, formats, defaults and conservative refusals. Co-Authored-By: Codex Signed-off-by: kikashy --- README.md | 16 +- bin/jpdev | 30 +- config/env.example | 2 + docs/policy.md | 105 +++++- lib/clean.py | 635 ++++++++++++++++++++++++++++++++++++ templates/codex-brief.md | 4 +- tests/clean_test.py | 679 +++++++++++++++++++++++++++++++++++++++ tests/run.sh | 39 ++- 8 files changed, 1502 insertions(+), 8 deletions(-) create mode 100644 lib/clean.py create mode 100644 tests/clean_test.py diff --git a/README.md b/README.md index eb9f217..4352be5 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,8 @@ the same name overrides it. `config/env.example` documents each setting with its lines of a failure. The exit status is the number of failures. - `jpdev mutants --row TEXT [--row TEXT]...`: runs the manifest's `mutants` command on a scratch copy of the clone (`.git` left out, `node_modules` linked) and prints caught or not caught per - row. + row. Clone-local scratch, temporary storage and the copy destination are excluded; their source + parents and siblings are retained. - `jpdev record post (--pr N | --issue N) --file FILE [--repo OWNER/NAME] [--dry-run]`: reduces the scratch and home paths to `` and ``, refuses to post when any local path remains, a line matches the name patterns, or the name-pattern file is missing or holds a @@ -106,6 +107,16 @@ the same name overrides it. `config/env.example` documents each setting with its `git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin BRANCH`, so the push uses the `gh` login and no stored credential. It refuses `--force`, `-f` and a `+` refspec. +- `jpdev clean [--root DIR]... [--older-than DURATION] [--yes]`: dry-run cleanup of explicit + session disposal seals; no roots means no deletion. Enroll an empty session root with + `jpdev clean init --root DIR --session TOKEN`, then attest completed disposable paths with + `jpdev clean seal --root DIR --session TOKEN --path REL --kind KIND` (worktree, review, scratch, + transcript or cache). Scratch also needs `--base BASE --removed-worktree OLD`; shared caches need + `--base BASE`. The default age is seven days (`7d`); durations use integer s/m/h/d/w units. + `.jpdev-clean-keep` holds exact relative reserves. JSON Lines reports candidates, sizes, reasons, + outcomes and totals; refusals exit nonzero. Linux host process visibility, disposal inventory, + Git/remote/PR evidence and all deletion checks remain mandatory with `--yes`. See + [Scratch and cleanup](docs/policy.md#scratch-and-cleanup) for setup, formats and limitations. - `jpdev worktree add BASE BRANCH [DIR]`, `jpdev worktree list BASE`, and `jpdev worktree remove DIR [--force]`: manage sibling worktrees. Removal refuses dirty or unpushed work unless forced. - `jpdev carry --pr N --model M --effort low|medium|high [--dir CLONE] [--worktree BASE]`: clone @@ -136,7 +147,8 @@ the same name overrides it. `config/env.example` documents each setting with its asked for was done. `--sandbox danger-full-access` is an explicit option only. Worktree runs share caches below `BASE/.jpdev-shared`; `--small` enforces the manifest handover paths and line bound after the run; `--resume` continues the named thread. Every detected thread id is written - to `.jpdev/output/thread`. A directory passed to `--add` adds every non-ignored file below it, + to `.jpdev/output/thread`. The builder always receives an existing `TMPDIR` beneath its effective + `JPDEV_SCRATCH`, replacing any inherited temporary directory. A directory passed to `--add` adds every non-ignored file below it, except the handover files and caches, and is refused with its path when nothing remains to add. - `jpdev codex review --brief FILE --model M --effort E --dir CLONE [--pr N | --issue N] [--repo OWNER/NAME] [--out FILE] [--sandbox read-only|workspace-write] [--network]`: checks the brief and diff --git a/bin/jpdev b/bin/jpdev index f9b910d..28bc2c0 100755 --- a/bin/jpdev +++ b/bin/jpdev @@ -22,6 +22,12 @@ usage: jpdev COMMAND [ARGS] name-pattern file, then post the file push [BRANCH] push BRANCH (default: the current branch) to origin with the gh credential helper and no stored credential; refuses --force and -f + clean [--root DIR]... [--older-than DURATION] [--yes] + inspect explicit session disposal seals; dry run by default + clean init --root DIR --session TOKEN + clean seal --root DIR --session TOKEN --path REL --kind KIND + [--base DIR] [--removed-worktree DIR] + enroll an empty session root, then seal session-created disposable state worktree add BASE BRANCH [DIR] worktree list BASE worktree remove DIR [--force] @@ -188,13 +194,28 @@ cmd_mutants() { copy=$JPDEV_SCRATCH/mutants/$(jpdev_clone_key "$root")-$$ mkdir -p "$copy" trap 'rm -rf -- "${copy:?}"' EXIT - rsync -a --exclude .git --exclude node_modules --exclude .venv "$root/" "$copy/" || jpdev_die "mutants: the scratch copy failed" - local found rel + # A builder's scratch and TMPDIR can be below the clone. Never copy generated state + # or the destination back into itself; preserve their parents and ordinary source siblings. + local exclude=(--exclude .git --exclude node_modules --exclude .venv) prune=(-path "$root/.git") + local generated generated_real escaped rel + for generated in "$copy" "$JPDEV_SCRATCH" "${TMPDIR:-}"; do + [ -d "$generated" ] || continue + generated_real=$(cd "$generated" && pwd -P) || jpdev_die "mutants: cannot resolve generated directory" + if [ "$generated_real" != "$root" ] && [[ $generated_real = "$root/"* ]]; then + rel=${generated_real#"$root"/} + escaped=${rel//\\/\\\\}; escaped=${escaped//\*/\\*}; escaped=${escaped//\?/\\?}; escaped=${escaped//\[/\\[} + exclude+=(--exclude "/$escaped/") + escaped=${generated_real//\\/\\\\}; escaped=${escaped//\*/\\*}; escaped=${escaped//\?/\\?}; escaped=${escaped//\[/\\[} + prune+=(-o -path "$escaped") + fi + done + rsync -a "${exclude[@]}" "$root/" "$copy/" || jpdev_die "mutants: the scratch copy failed" + local found while IFS= read -r found; do rel=${found#"$root"/} mkdir -p "$copy/$(dirname "$rel")" ln -s "$found" "$copy/$rel" - done < <(find "$root" -name node_modules -type d -not -path '*/node_modules/*' -not -path "$root/.git/*" -prune -print) + done < <(find "$root" \( "${prune[@]}" \) -prune -o -name node_modules -type d -prune -print) local args="" row for row in "${rows[@]}"; do args+=$(printf ' --row %q' "$row"); done local out=$copy/.jpdev-mutants.out @@ -654,6 +675,8 @@ codex_run() { [ -n "${GOFLAGS:-}" ] || export GOFLAGS=$codex_goflags [ -n "${npm_config_cache:-}" ] || export npm_config_cache=$codex_npm_cache [ -n "${XDG_CACHE_HOME:-}" ] || export XDG_CACHE_HOME=$codex_xdg_cache + export TMPDIR=$codex_scratch/tmp + mkdir -p "$TMPDIR" || exit 2 codex "${args[@]}" < /dev/null > "$log" 2>&1 ) local status=$? @@ -998,6 +1021,7 @@ case ${1:-} in mutants) shift; cmd_mutants "$@" ;; record) shift; cmd_record "$@" ;; push) shift; cmd_push "$@" ;; + clean) shift; exec python3 -I "$kit/lib/clean.py" "$@" ;; worktree) shift; cmd_worktree "$@" ;; carry) shift; cmd_carry "$@" ;; codex) shift; cmd_codex "$@" ;; diff --git a/config/env.example b/config/env.example index 97a512c..b9bd939 100644 --- a/config/env.example +++ b/config/env.example @@ -4,6 +4,8 @@ # `jpdev env` prints the effective values and where each came from. # Where jpdev keeps its logs, scratch copies for `jpdev mutants`, and Go build caches. +# Builder TMPDIR is always $JPDEV_SCRATCH/tmp; an inherited TMPDIR is replaced. +# Cleanup roots are explicit CLI arguments, never inferred from this setting. #JPDEV_SCRATCH="${XDG_STATE_HOME:-$HOME/.local/state}/jpdev" # The machine lock every Go step holds, so two sessions never run Go at once. diff --git a/docs/policy.md b/docs/policy.md index 91eb4e0..971f798 100644 --- a/docs/policy.md +++ b/docs/policy.md @@ -50,7 +50,8 @@ Before handover, a builder runs the self-checks in AGENTS.md on its own head. It touched check steps while iterating and runs the whole check list exactly once at the end. CI must be green before any review begins. `jpdev codex run` gives Codex clone-local scratch, lock, Go and npm caches and an XDG cache when those variables are empty, and creates their directories under -the clone. +the clone. Builder `TMPDIR` is always created under its effective scratch, including when an +external temporary directory was inherited. After a fix round that closed only MEDIUM findings, the review re-read may use the cheaper tier. After a HIGH finding, the re-read uses the same tier as the review of record. A HIGH is fixed in the @@ -135,3 +136,105 @@ verified before the release is announced. Tracked files, commit messages, pull request bodies and issue comments carry no machine-local paths and no personal names. A person is referred to by role or by handle where a reference is needed. Plain, factual wording: what changed, what was checked, what was not. + +## Scratch and cleanup + +A session removes only state it created. Use `jpdev clean` after the pull request it built or +reviewed merges, after release, and when disk use exceeds 85%. Never remove another session's +data, a live checkout, or an operator reserve. Operator approval authorizes one narrow +`jpdev clean --root DIR --older-than DURATION --yes` invocation; all tool checks still apply. +Inspect the dry run first. Approval does not transfer ownership of existing state. + +Before creating disposable state, initialize an empty, session-created root with +`jpdev clean init --root DIR --session TOKEN`. TOKEN is 8–80 letters, digits, underscores or +hyphens, beginning with a letter or digit; choose a unique session prefix. Init creates DIR +if its parent exists, and refuses a nonempty directory. Put the session's clone, worktrees, +review rooms, logs and scratch beneath it. Configure `JPDEV_SCRATCH` and other output locations +there as needed. The wrapper creates builder `TMPDIR` beneath its effective scratch, replacing +an inherited temporary directory. Temporary files stay in scratch, never loose in the system +temporary directory; temporary directories carry the session prefix. + +After the session finishes writing a disposable path, explicitly attest its ownership and +contents with `jpdev clean seal --root DIR --session TOKEN --path REL --kind KIND`. This is an +operator/session assertion that every included file, including untracked, ignored and review +evidence, was created by this session and is disposable at the requested milestone. Never seal +someone else's files. Sealing historical unmarked directories is forbidden; init cannot adopt +their nonempty root. Tools do not automatically seal or infer ownership from a name or UID. +A changed path needs a new explicit seal after its evidence has been assessed. + +Kinds are `worktree`, `review` (including reviewer clones), `transcript` (a log directory +or an individual regular log/transcript file), `scratch`, and `cache`. Scratch for a removed +worktree also requires `--base BASE --removed-worktree OLD`; shared base caches require +`--base BASE`. BASE and OLD are absolute in the stored seal; supply all relevant roots explicitly +to cleanup. A whole worktree seal may include its own scratch, locks and caches. Seal separate +scratch/lock/cache directories with `scratch` when they outlive that worktree. Base +`.jpdev-shared` contents are eligible only once no linked worktree directory remains. Unknown, +inaccessible, symlinked or out-of-root worktree registration evidence causes refusal. Stale Git +registration entries for verified absent paths beneath authorized, traversable parents are +retained and do not count as active users. Cleanup does not prune Git administrative records or +remove branch refs: a broad prune could affect another session. + +`jpdev clean [--root DIR]... [--older-than DURATION] [--yes]` discovers only disposal seals, +never recursively guesses candidates from directory names. There are no implicit roots; no flags +means no deletion. Missing roots are reported safely. It defaults to a dry run and seven days; +DURATION is a nonnegative integer followed by `s`, `m`, `h`, `d` or `w`. Age is measured from the +newest seal timestamp, candidate directory/file mtime and inventoried content mtime. Thus a new +seal must age even if its logs are old. There is no size cutoff: sessions can seal large logs +individually. Paths are root-relative, exact, and cannot contain empty, dot or parent components. +Overlapping candidate seals are refused. The ownership metadata and keep file are protected. + +Place exact root-relative reserves in `DIR/.jpdev-clean-keep`, one per line; blank lines and +`#` comments are allowed. A reserve protects itself, descendants, and any candidate containing it. +Invalid or symlinked keep files cause refusal. Roots, the invoking checkout, home, base checkouts +and any candidate containing an explicitly authorized root are protected. Keep all live checkouts +and operator reserves in this file even when their names look disposable. + +Cleanup requires the recorded inode identities and complete SHA-256/stat inventory to match, +including all ignored and untracked files. A clean tracked diff is insufficient. Git worktrees +must have no tracked or staged changes and no unpublished commits reachable from HEAD or any +local ref, including tags, stashes and other branches. Git metadata must lie beneath an explicitly +authorized root. Publication is checked against current `git ls-remote --refs origin`, without +fetching. Locally unavailable remote objects cause refusal. Attached branches require a verified +origin upstream containing HEAD and a GitHub PR with matching branch and HEAD in CLOSED or +MERGED state. Detached worktrees and detached reviewer clones use verified remote reachability +without a PR. Reviewer clones with attached branches use the same upstream and PR checks. +A reviewer clone with any registered linked worktree remains protected. Git, remote and PR +evidence must remain available; an unavailable tool, network or +authentication causes refusal. `gh` is needed only for attached branches. + +Deletion requires Linux procfs with the initial host PID namespace (Linux initial namespace inode +4026531836), the host mount namespace and root view, and readable process evidence for every +visible process. Processes in another mount namespace or root view cause refusal because path +aliases cannot establish inactivity. A container, sandbox, PID namespace, hidden process view or permission restriction +cannot establish host inactivity and is refused. Process cwd, root, executable, open descriptors, +mappings, command arguments and declared scratch/cache environment paths are checked. Live +processes are reported by PID and never stopped. Symlinks, special files, nested repositories, +mount crossings and changed ownership are refused. The tool requires directory descriptors, +`O_NOFOLLOW` and POSIX locking; unsupported platforms fail closed. + +Before removal the tool repeats inspection, moves the candidate to a private sibling quarantine +name, and rechecks its full inventory, Git state, keeps, root and parent identities and process +activity. Removal uses directory descriptors, never recursive path-following deletion. Each +entry's identity and inventory is checked during removal. External linked-worktree refs, HEAD +and index are rechecked during removal as well. Changes or failures stop that candidate; +remaining content is restored to its original name when vacant, otherwise retained under +`.jpdev-clean-removing-` for operator inspection. A refusal can leave a partially removed +candidate; the remaining evidence is retained and requires assessment before another seal. +These checks address ordinary concurrent activity; they do not isolate a malicious process with +the same user identity, which can edit the tool or its ownership records. + +Cleanup prints JSON Lines: candidate records contain `root`, root-relative `path`, logical `bytes`, +`reason` and `outcome`; `--yes` adds per-path `removed` or `refused` outcomes. The final record gives +`candidates`, logical `bytes`, `removed`, `refused` and `mode`. Totals count inspected candidates, +including candidates later refused during removal; they do not estimate physical reclaimed space. +No candidate or root failure is silently skipped. Exit 0 means no refusal (including a no-op or +absent root), 1 means cleanup reported a refusal/removal failure, and 2 means invalid arguments or +init/seal refusal. Init and seal print a short confirmation, rather than cleanup JSON Lines. + +On-disk metadata is version 1 JSON in `DIR/.jpdev-clean`: `owner.json` records session token and +root device/inode; each seal is named by SHA-256 of its relative path and records version, session, +path, kind, candidate device/inode, seal time, content inventory, and optional base/removed paths. +Inventory keys are relative paths (empty for a standalone transcript file); values hold device, +inode, mode, logical size, nanosecond mtime/ctime, and SHA-256 for regular files. Records must be +regular, session-user-owned files, not writable by group/other, and at most 16 MiB. Root ownership +records and reserves are retained. They are local session evidence, never repository content. diff --git a/lib/clean.py b/lib/clean.py new file mode 100644 index 0000000..99e62ae --- /dev/null +++ b/lib/clean.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 +"""Explicit session disposal seals and conservative, descriptor-relative cleanup (Linux).""" +import argparse +import contextlib +import fcntl +import hashlib +import json +import os +from pathlib import PurePosixPath +import re +import stat +import subprocess +import sys +import time +import uuid + +META = '.jpdev-clean' +KINDS = ('worktree', 'review', 'scratch', 'transcript', 'cache') +FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC + + +class Refusal(Exception): + pass + + +def require(condition, reason): + if not condition: + raise Refusal(reason) + + +def relative(path): + parts = PurePosixPath(path).parts + require(isinstance(path, str) and path and not path.startswith('/') and + all(p not in ('.', '..', META, '.jpdev-clean-keep') for p in parts) and + '/'.join(parts) == path, 'invalid or protected relative path') + return parts + + +def identity(st): + return [st.st_dev, st.st_ino] + + +def stamp(st): + return [st.st_dev, st.st_ino, st.st_mode, st.st_size, st.st_mtime_ns, st.st_ctime_ns] + + +@contextlib.contextmanager +def directory(path): + """Open every component without resolving symlinks, including ancestors of roots.""" + path = os.path.abspath(path) + fd = os.open('/', FLAGS) + try: + for component in path.split('/')[1:]: + new = os.open(component, FLAGS, dir_fd=fd) + os.close(fd) + fd = new + yield fd + finally: + os.close(fd) + + +@contextlib.contextmanager +def descend(fd, parts): + child = os.dup(fd) + try: + for part in parts: + new = os.open(part, FLAGS, dir_fd=child) + os.close(child) + child = new + yield child + finally: + os.close(child) + + +def read_json(fd, name): + source = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + with os.fdopen(source) as stream: + st = os.fstat(stream.fileno()) + require(stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid() and + not st.st_mode & 0o022 and st.st_size <= 16 * 1024 * 1024, + 'unsafe ownership record') + value = json.load(stream) + require(isinstance(value, dict), 'ownership record must be an object') + return value + + +def write_json(fd, name, value): + temp = '.new-' + uuid.uuid4().hex + source = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd) + with os.fdopen(source, 'w') as stream: + json.dump(value, stream, sort_keys=True) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + os.rename(temp, name, src_dir_fd=fd, dst_dir_fd=fd) + + +def file_snapshot(fd, name): + before = os.stat(name, dir_fd=fd, follow_symlinks=False) + require(stat.S_ISREG(before.st_mode) and before.st_uid == os.getuid(), 'file ownership/type unavailable') + source = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + with os.fdopen(source, 'rb') as stream: + require(stamp(os.fstat(stream.fileno())) == stamp(before), 'file replaced: ' + name) + hasher = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b''): + hasher.update(block) + require(stamp(os.fstat(stream.fileno())) == stamp(before), 'file changed: ' + name) + require(stamp(os.stat(name, dir_fd=fd, follow_symlinks=False)) == stamp(before), 'file replaced: ' + name) + return stamp(before) + [hasher.hexdigest()] + + +def inventory(fd, prefix=''): + """Never follow symlinks or mounts; reject special files and nested repositories.""" + result = {} + device = os.fstat(fd).st_dev + for name in sorted(os.listdir(fd)): + key = prefix + name + before = os.stat(name, dir_fd=fd, follow_symlinks=False) + require(before.st_uid == os.getuid(), 'content ownership unavailable: ' + key) + require(before.st_dev == device, 'nested mount: ' + key) + require(not stat.S_ISLNK(before.st_mode), 'symlink: ' + key) + require(stat.S_ISREG(before.st_mode) or stat.S_ISDIR(before.st_mode), 'special file: ' + key) + if stat.S_ISDIR(before.st_mode): + require(name != '.git' or not prefix, 'nested repository: ' + key) + with descend(fd, (name,)) as child: + result[key] = stamp(before) + result.update(inventory(child, key + '/')) + else: + require(name != '.git' or not prefix, 'nested repository: ' + key) + result[key] = file_snapshot(fd, name) + require(stamp(os.stat(name, dir_fd=fd, follow_symlinks=False)) == stamp(before), 'content changed: ' + key) + return result + + +def run(args, cwd): + try: + completed = subprocess.run(args, cwd=cwd, stdin=subprocess.DEVNULL, capture_output=True, + text=True, timeout=30, env={**os.environ, 'GIT_OPTIONAL_LOCKS': '0', + 'GIT_TERMINAL_PROMPT': '0'}) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Refusal('verification unavailable: ' + args[0]) from exc + require(completed.returncode == 0, 'verification failed: ' + ' '.join(args[:3])) + return completed.stdout.strip() + + +def within(path, roots): + if not isinstance(path, str) or not os.path.isabs(path) or os.path.normpath(path) != path: + return False + return any(path == root or path.startswith(root + os.sep) for root in roots) + + +def mount_check(path): + require(sys.platform == 'linux', 'mount verification requires Linux procfs') + with open('/proc/self/mountinfo') as stream: + for line in stream: + fields = line.split() + require(len(fields) >= 6, 'mount evidence unavailable') + mountpoint = re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), fields[4]) + require(not within(mountpoint, [path]), 'mounted content: ' + mountpoint) + + +def process_check(paths): + """A namespaced or unreadable /proc is insufficient evidence of host inactivity.""" + require(sys.platform == 'linux', 'complete process verification requires Linux procfs') + try: + # Linux's initial PID namespace inode. A nested namespace can hide processes holding files. + require(os.readlink('/proc/self/ns/pid') == 'pid:[4026531836]' and + os.readlink('/proc/1/ns/pid') == 'pid:[4026531836]', + 'process visibility incomplete: initial host PID namespace required') + require(os.readlink('/proc/self/ns/mnt') == os.readlink('/proc/1/ns/mnt') and + identity(os.stat('/')) == identity(os.stat('/proc/1/root')), + 'process visibility incomplete: host mount/root view required') + for pid in os.listdir('/proc'): + if not pid.isdigit() or int(pid) == os.getpid(): + continue + try: + with open('/proc/' + pid + '/stat') as stream: + flags = int(stream.read().rsplit(')', 1)[1].split()[6]) + if flags & 0x00200000: # PF_KTHREAD has no userspace cwd/executable. + continue + require(os.readlink('/proc/' + pid + '/ns/mnt') == os.readlink('/proc/self/ns/mnt') and + identity(os.stat('/proc/' + pid + '/root')) == identity(os.stat('/')), + 'process visibility incomplete for PID ' + pid) + links = [os.readlink('/proc/' + pid + '/' + item) for item in ('cwd', 'root', 'exe')] + for name in os.listdir('/proc/' + pid + '/fd'): + try: + links.append(os.readlink('/proc/' + pid + '/fd/' + name)) + except FileNotFoundError: + continue + with open('/proc/' + pid + '/cmdline', 'rb') as stream: + argv = [p.decode(errors='surrogateescape') for p in stream.read().split(b'\0') if p] + links.extend(os.path.abspath(os.path.join(links[0], p.split('=', 1)[-1])) for p in argv) + with open('/proc/' + pid + '/maps') as stream: + links.extend(line.split(None, 5)[5].strip() for line in stream if len(line.split(None, 5)) == 6 and line.split(None, 5)[5].startswith('/')) + with open('/proc/' + pid + '/environ', 'rb') as stream: + for item in stream.read().split(b'\0'): + key, _, value = item.partition(b'=') + if key in (b'TMPDIR', b'JPDEV_SCRATCH', b'GOCACHE', b'GOMODCACHE', b'npm_config_cache', b'XDG_CACHE_HOME') and value: + links.append(os.path.abspath(os.path.join(links[0], value.decode(errors='surrogateescape')))) + for link in links: + if link.endswith(' (deleted)'): + link = link[:-10] + require(not within(link, paths), 'live PID ' + pid + ': ' + link) + except FileNotFoundError: + # A process may exit, but a kernel thread without an exe is still present. + require(not os.path.exists('/proc/' + pid), 'process evidence unavailable for PID ' + pid) + except OSError as exc: + raise Refusal('process visibility unavailable: ' + str(exc)) from exc + + +def keep_check(rootfd, path): + try: + source = os.open('.jpdev-clean-keep', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=rootfd) + except FileNotFoundError: + return + with os.fdopen(source) as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), 'invalid keep file') + for line in stream: + entry = line.strip() + if not entry or entry.startswith('#'): + continue + relative(entry) + require(not (path == entry or path.startswith(entry + '/') or entry.startswith(path + '/')), + 'operator keep: ' + entry) + + +def git_layout(path, roots): + # Validate Git's indirection before invoking Git, which would otherwise discover metadata + # beyond the authorized roots. Explicit arguments keep later Git calls on this evidence. + with directory(path) as fd: + marker = os.stat('.git', dir_fd=fd, follow_symlinks=False) + if stat.S_ISDIR(marker.st_mode): + gitdir = path + '/.git' + else: + require(stat.S_ISREG(marker.st_mode), 'invalid Git marker') + source = os.open('.git', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + with os.fdopen(source) as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), 'invalid Git marker') + pointer = stream.read(4096).strip() + require(pointer.startswith('gitdir: ') and '\n' not in pointer, 'invalid Git marker') + gitdir = os.path.abspath(os.path.join(path, pointer[8:])) + require(within(gitdir, roots), 'Git evidence outside authorized roots') + with directory(gitdir) as fd: + try: + source = os.open('commondir', os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + except FileNotFoundError: + common = gitdir + else: + with os.fdopen(source) as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), 'invalid Git common directory marker') + common = os.path.abspath(os.path.join(gitdir, stream.read(4096).strip())) + require(within(common, roots), 'Git evidence outside authorized roots') + with directory(common): + pass + return common, gitdir + + +def git_metadata(gitdir): + result = {} + with directory(gitdir) as fd: + for name in ('HEAD', 'index', 'commondir'): + try: + result[name] = file_snapshot(fd, name) + except FileNotFoundError: + result[name] = None + return result + + +def git_check(path, roots, kind): + common, gitdir = git_layout(path, roots) + require(common != gitdir or kind == 'review', 'live base checkout is protected') + def gitrun(args): + return run(['git', '--git-dir=' + gitdir, '--work-tree=' + path, + '-c', 'core.fsmonitor=false', *args], path) + worktrees = gitrun(['worktree', 'list', '--porcelain', '-z']).split('\0') + for item in worktrees: + if item.startswith('worktree '): + other = item[9:] + require(common != gitdir or other == path, 'base still has registered worktrees') + require(not (other != path and other.startswith(path + '/')), 'nested worktree: ' + other) + require(not gitrun(['diff', '--name-only', 'HEAD', '--']), 'tracked changes') + require(not gitrun(['diff', '--cached', '--name-only', '--']), 'tracked index changes') + # The explicit inventory seal is the disposal evidence for untracked and ignored files. + gitrun(['status', '--porcelain=v1', '--untracked-files=all', '--ignored']) + remote = gitrun(['ls-remote', '--refs', 'origin']) + published = dict(line.split('\t')[::-1] for line in remote.splitlines()) + require(published, 'remote publication evidence unavailable') + commits = sorted(set(published.values())) + require(not gitrun(['rev-list', '--all', 'HEAD', '--not', *commits]), + 'unpublished commits (including other refs)') + branch = gitrun(['branch', '--show-current']) + head = gitrun(['rev-parse', 'HEAD']) + if branch: + upstream = gitrun(['for-each-ref', '--format=%(upstream:remotename) %(upstream:remoteref)', + 'refs/heads/' + branch]).split() + require(len(upstream) == 2 and upstream[0] == 'origin' and upstream[1] in published, + 'upstream publication unavailable') + require(gitrun(['merge-base', head, published[upstream[1]]]) == head, + 'upstream does not contain HEAD') + try: + prs = json.loads(run(['gh', 'pr', 'list', '--state', 'all', '--head', branch, + '--json', 'state,headRefOid,headRefName', '--limit', '100', + '--repo', gitrun(['remote', 'get-url', 'origin'])], path)) + except (ValueError, TypeError) as exc: + raise Refusal('PR evidence unavailable') from exc + require(isinstance(prs, list) and any(isinstance(pr, dict) and + pr.get('state') in ('MERGED', 'CLOSED') and pr.get('headRefName') == branch and + pr.get('headRefOid') == head for pr in prs), 'PR is not verified merged or closed') + return {'common': common, 'gitdir': gitdir, 'head': head, 'remote': remote, 'branch': branch, + 'refs': gitrun(['for-each-ref', '--format=%(refname) %(objectname)']), + 'metadata': git_metadata(gitdir)} + + +class Root: + def __init__(self, path, fd): + self.path, self.fd = path, fd + self.identity = identity(os.fstat(fd)) + require(os.fstat(fd).st_uid == os.getuid() and not os.fstat(fd).st_mode & 0o022, 'unsafe root ownership or permissions') + self.meta = os.open(META, FLAGS, dir_fd=fd) + require(os.fstat(self.meta).st_uid == os.getuid() and not os.fstat(self.meta).st_mode & 0o022, 'unsafe ownership directory permissions') + self.meta_identity = identity(os.fstat(self.meta)) + self.owner = read_json(self.meta, 'owner.json') + require(self.owner.get('version') == 1 and self.owner.get('root') == self.identity and + isinstance(self.owner.get('session'), str), 'root ownership unavailable') + + def recheck(self): + with directory(self.path) as now: + require(identity(os.fstat(now)) == self.identity, 'root replaced') + require(os.fstat(now).st_uid == os.getuid() and not os.fstat(now).st_mode & 0o022, 'root ownership or permissions changed') + with descend(now, (META,)) as meta: + require(identity(os.fstat(meta)) == self.meta_identity, 'ownership directory replaced') + require(os.fstat(meta).st_uid == os.getuid() and not os.fstat(meta).st_mode & 0o022, 'ownership directory permissions changed') + require(read_json(self.meta, 'owner.json') == self.owner, 'ownership changed') + + def records(self): + for name in sorted(os.listdir(self.meta)): + if re.fullmatch(r'[a-f0-9]{64}\.json', name): + yield name, read_json(self.meta, name) + + +def inspect(root, record, roots, age, processes=process_check): + root.recheck() + path = record.get('path') + parts = relative(path) + require(record.get('version') == 1 and record.get('session') == root.owner['session'] and + record.get('kind') in KINDS, 'ownership/disposal record unavailable') + seal_name = hashlib.sha256(path.encode()).hexdigest() + '.json' + require(read_json(root.meta, seal_name) == record, 'disposal seal changed') + keep_check(root.fd, path) + full = root.path + '/' + path + mount_check(full) + require(not within(os.getcwd(), [full]) and full != os.path.expanduser('~') and + not any(r == full or r.startswith(full + '/') for r in roots), 'protected live checkout or root') + with descend(root.fd, parts[:-1]) as parent: + st = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) + require(identity(st) == record.get('identity'), 'candidate replaced') + require(st.st_uid == os.getuid(), 'candidate ownership changed') + require(stat.S_ISDIR(st.st_mode) or (record['kind'] == 'transcript' and stat.S_ISREG(st.st_mode)), 'candidate must be a real directory or transcript file') + if stat.S_ISDIR(st.st_mode): + with descend(parent, parts[-1:]) as child: + contents = inventory(child) + else: + contents = {'': file_snapshot(parent, parts[-1])} + require(contents == record.get('inventory'), 'content changed or lacks explicit disposal evidence') + newest = max([record.get('sealed', time.time()), st.st_mtime] + [v[4] / 1e9 for v in contents.values()]) + require(time.time() - newest >= age, 'younger than age threshold') + # A repository marker is allowed only at the candidate root, and only for Git-aware classes. + is_git = '.git' in contents + git = None + if record['kind'] in ('worktree', 'review') and is_git: + git = git_check(full, roots, record['kind']) + else: + require(not any(PurePosixPath(p).name == '.git' for p in contents), 'nested worktree or repository') + require(record['kind'] != 'worktree', 'worktree evidence missing') + if record['kind'] in ('scratch', 'cache'): + base = record.get('base') + require(isinstance(base, str) and within(base, roots), 'base evidence outside authorized roots or missing') + with directory(base): + pass + common, gitdir = git_layout(base, roots) + require(common == gitdir, 'base must be the main checkout') + lines = run(['git', '--git-dir=' + gitdir, '--work-tree=' + base, + 'worktree', 'list', '--porcelain'], base).splitlines() + users = [] + for line in lines: + if not line.startswith('worktree '): + continue + user = line[9:] + require(within(user, roots), 'worktree evidence outside authorized roots') + try: + with directory(user): + pass + users.append(user) + except FileNotFoundError: + # Absence is established only under an authorized, traversable parent. + require(within(user, roots), 'missing worktree evidence outside authorized roots') + with directory(os.path.dirname(user)): + pass + if record['kind'] == 'cache': + require(users == [base], 'base still has worktree users') + else: + removed = record.get('removed') + require(isinstance(removed, str) and within(removed, roots), 'associated worktree path unavailable') + with directory(os.path.dirname(removed)) as removed_parent: + try: + os.stat(os.path.basename(removed), dir_fd=removed_parent, follow_symlinks=False) + absent = False + except FileNotFoundError: + absent = True + require(absent and removed not in users, 'associated worktree is not removed') + processes([full] + ([git['gitdir']] if git else [])) + return contents, git + + +def erase(fd, expected, prefix='', before_unlink=None): + require(sorted(os.listdir(fd)) == sorted(p[len(prefix):] for p in expected + if p.startswith(prefix) and '/' not in p[len(prefix):]), 'directory changed during removal') + for name in sorted(os.listdir(fd)): + key = prefix + name + before = os.stat(name, dir_fd=fd, follow_symlinks=False) + require(stamp(before) == expected[key][:6], 'entry changed during removal: ' + key) + if stat.S_ISDIR(before.st_mode): + with descend(fd, (name,)) as child: + erase(child, expected, key + '/', before_unlink) + require(identity(os.stat(name, dir_fd=fd, follow_symlinks=False)) == identity(before), 'directory replaced during removal') + if before_unlink: + before_unlink() + os.rmdir(name, dir_fd=fd) + else: + if before_unlink: + before_unlink() + require(stamp(os.stat(name, dir_fd=fd, follow_symlinks=False)) == expected[key][:6], 'file replaced during removal') + os.unlink(name, dir_fd=fd) + + +def remove(root, record, roots, age, processes=process_check): + contents, git = inspect(root, record, roots, age, processes) + parts = relative(record['path']) + token = '.jpdev-clean-removing-' + uuid.uuid4().hex + with descend(root.fd, parts[:-1]) as parent: + root.recheck() + # Move into a private sibling name, then verify the moved inode and complete inventory. + os.rename(parts[-1], token, src_dir_fd=parent, dst_dir_fd=parent) + moved = root.path + '/' + '/'.join((*parts[:-1], token)) + try: + mount_check(moved) + st = os.stat(token, dir_fd=parent, follow_symlinks=False) + require(identity(st) == record['identity'], 'candidate replaced during quarantine') + if stat.S_ISREG(st.st_mode): + moved_file = file_snapshot(parent, token) + require(moved_file[:5] + moved_file[6:] == contents[''][:5] + contents[''][6:], 'content changed during quarantine') + root.recheck() + require(read_json(root.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json') == record, 'disposal seal changed') + keep_check(root.fd, record['path']) + processes([moved, root.path + '/' + record['path']]) + with descend(root.fd, parts[:-1]) as current_parent: + require(identity(os.fstat(current_parent)) == identity(os.fstat(parent)), 'parent replaced') + require(file_snapshot(parent, token) == moved_file, 'file replaced during removal') + os.unlink(token, dir_fd=parent) + return + with descend(parent, (token,)) as child: + require(inventory(child) == contents, 'content changed during quarantine') + if git: + expected_git = {key: (moved + val[len(root.path + '/' + record['path']):] + if key in ('common', 'gitdir') and + within(val, [root.path + '/' + record['path']]) else val) + for key, val in git.items()} + after = git_check(moved, roots, record['kind']) + require(after == expected_git, 'Git state changed during quarantine') + keep_check(root.fd, record['path']) + root.recheck() + def check(): + root.recheck() + mount_check(moved) + if git and not within(after['gitdir'], [moved]): + require(git_metadata(after['gitdir']) == after['metadata'] and + run(['git', '--git-dir=' + after['gitdir'], 'for-each-ref', + '--format=%(refname) %(objectname)'], root.path) == after['refs'], + 'Git state changed during removal') + require(read_json(root.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json') == record, 'disposal seal changed') + keep_check(root.fd, record['path']) + with descend(root.fd, parts[:-1]) as current_parent: + require(identity(os.fstat(current_parent)) == identity(os.fstat(parent)), 'parent replaced') + processes([moved, root.path + '/' + record['path']] + ([git['gitdir']] if git else [])) + check() + erase(child, contents, before_unlink=check) + require(identity(os.stat(token, dir_fd=parent, follow_symlinks=False)) == record['identity'], 'quarantine replaced') + os.rmdir(token, dir_fd=parent) + except BaseException: + if not os.path.lexists(root.path + '/' + record['path']): + os.rename(token, parts[-1], src_dir_fd=parent, dst_dir_fd=parent) + raise + # Git administration is deliberately retained: pruning can touch unrelated worktrees. + + +def duration(text): + match = re.fullmatch(r'(\d+)([smhdw])', text) + if not match: + raise argparse.ArgumentTypeError('duration must be an integer followed by s, m, h, d or w') + return int(match[1]) * dict(s=1, m=60, h=3600, d=86400, w=604800)[match[2]] + + +def initialize(path, session): + require(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_-]{7,79}', session), 'session token must be 8..80 letters, digits, underscores or hyphens') + if not os.path.exists(path): + os.mkdir(path, 0o700) + with directory(path) as fd: + require(not os.listdir(fd), 'init requires an empty session-created root') + require(os.fstat(fd).st_uid == os.getuid() and not os.fstat(fd).st_mode & 0o022, 'unsafe root ownership or permissions') + os.mkdir(META, 0o700, dir_fd=fd) + with descend(fd, (META,)) as meta: + write_json(meta, 'owner.json', {'version': 1, 'session': session, 'root': identity(os.fstat(fd))}) + + +def seal(root, args): + require(root.owner['session'] == args.session, 'session ownership mismatch') + parts = relative(args.path) + mount_check(root.path + '/' + args.path) + keep_check(root.fd, args.path) + with descend(root.fd, parts[:-1]) as parent: + st = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) + if args.kind == 'transcript' and stat.S_ISREG(st.st_mode): + contents = {'': file_snapshot(parent, parts[-1])} + else: + with descend(parent, parts[-1:]) as child: + contents = inventory(child) + ident = identity(st) + for _, existing in root.records(): + old = existing['path'] + require(not (old != args.path and (old.startswith(args.path + '/') or args.path.startswith(old + '/'))), + 'overlapping disposal records') + value = {'version': 1, 'session': args.session, 'path': args.path, 'kind': args.kind, + 'identity': ident, 'inventory': contents, 'sealed': time.time()} + if args.base: + value['base'] = os.path.abspath(args.base) + if args.removed_worktree: + value['removed'] = os.path.abspath(args.removed_worktree) + write_json(root.meta, hashlib.sha256(args.path.encode()).hexdigest() + '.json', value) + + +def parser(): + result = argparse.ArgumentParser(description=__doc__) + result.add_argument('action', nargs='?', choices=('init', 'seal')) + result.add_argument('--root', action='append', default=[], help='explicit root; repeat to authorize Git evidence roots') + result.add_argument('--older-than', type=duration, default=7 * 86400, metavar='DURATION') + result.add_argument('--yes', action='store_true', help='remove verified candidates (default: dry run)') + result.add_argument('--session') + result.add_argument('--path') + result.add_argument('--kind', choices=KINDS) + result.add_argument('--base') + result.add_argument('--removed-worktree') + return result + + +def main(argv=None): + args = parser().parse_args(argv) + roots = list(dict.fromkeys(os.path.abspath(p) for p in args.root)) + if args.action: + require(len(roots) == 1 and args.session and not args.yes, 'init/seal require one root and a session, without --yes') + if args.action == 'init': + initialize(roots[0], args.session) + print('initialized session cleanup root') + return 0 + require(args.path and args.kind, 'seal requires --path and --kind') + else: + require(not any((args.session, args.path, args.kind, args.base, args.removed_worktree)), 'seal options require seal action') + total = size = removed = refused = 0 + for path in roots: + try: + with directory(path) as fd: + try: + root = Root(path, fd) + except FileNotFoundError: + if args.action: + raise Refusal('root has no session ownership record') + print(json.dumps({'root': path, 'outcome': 'refused', 'reason': 'unmarked root; no ownership/disposal evidence'})) + refused += 1 + continue + try: + fcntl.flock(root.meta, fcntl.LOCK_EX | fcntl.LOCK_NB) + if args.action: + seal(root, args) + print('sealed explicit session disposal inventory') + return 0 + records = list(root.records()) + for name, record in records: + try: + contents, _ = inspect(root, record, roots, args.older_than) + bytes_used = sum(v[3] for v in contents.values() if stat.S_ISREG(v[2])) + total += 1 + size += bytes_used + print(json.dumps({'root': path, 'path': record['path'], 'bytes': bytes_used, + 'reason': 'sealed ' + record['kind'], 'outcome': 'candidate'}), flush=True) + if args.yes: + remove(root, record, roots, args.older_than) + os.unlink(name, dir_fd=root.meta) + removed += 1 + print(json.dumps({'root': path, 'path': record['path'], 'outcome': 'removed'}), flush=True) + except (Refusal, OSError, ValueError, KeyError, TypeError) as exc: + refused += 1 + print(json.dumps({'root': path, 'path': record.get('path'), 'outcome': 'refused', 'reason': str(exc)}), flush=True) + # Show unmarked top-level state without recursively discovering or claiming it. + covered = {r['path'].split('/')[0] for _, r in records} + for entry in sorted(set(os.listdir(fd)) - covered - {META, '.jpdev-clean-keep'}): + refused += 1 + print(json.dumps({'root': path, 'path': entry, 'outcome': 'refused', 'reason': 'no explicit disposal seal'})) + finally: + os.close(root.meta) + except FileNotFoundError: + if args.action: + raise Refusal('session root or ownership evidence is absent') + try: + with directory(path): + pass + except FileNotFoundError: + print(json.dumps({'root': path, 'outcome': 'absent'})) + else: + refused += 1 + print(json.dumps({'root': path, 'outcome': 'refused', 'reason': 'ownership evidence disappeared'})) + except (Refusal, OSError, ValueError, KeyError, TypeError) as exc: + if args.action: + raise Refusal(str(exc)) from exc + refused += 1 + print(json.dumps({'root': path, 'outcome': 'refused', 'reason': str(exc)})) + print(json.dumps({'candidates': total, 'bytes': size, 'removed': removed, 'refused': refused, + 'mode': 'remove' if args.yes else 'dry-run'})) + return 1 if refused else 0 + + +if __name__ == '__main__': + try: + sys.exit(main()) + except (Refusal, OSError, ValueError, KeyError, TypeError) as error: + print('jpdev clean: refused: ' + str(error), file=sys.stderr) + sys.exit(2) diff --git a/templates/codex-brief.md b/templates/codex-brief.md index c2d4263..44b8712 100644 --- a/templates/codex-brief.md +++ b/templates/codex-brief.md @@ -2,7 +2,9 @@ ## Rules Read this repository's AGENTS.md and follow it. Work only in the current directory. Start no -sub-agents. Never use a pattern kill (`pkill`, `killall`). +sub-agents. Temporary files belong beneath `$JPDEV_SCRATCH` (including `$TMPDIR`), +never loose in the system temporary directory. Give temporary directories a session prefix. +Never use a pattern kill (`pkill`, `killall`). You cannot write `.git` here, so do not run `git commit`, `git push` or `gh`. Edit the files, run the repository's check list once (`scripts/dev/check.sh`, or `jpdev check`), and write the two diff --git a/tests/clean_test.py b/tests/clean_test.py new file mode 100644 index 0000000..99b3161 --- /dev/null +++ b/tests/clean_test.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python3 +"""Hand-built disposal fixtures. All deletions are confined to this run's temporary root.""" +import contextlib +import importlib.util +import io +import hashlib +import stat +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +KIT = Path(__file__).resolve().parents[1] +SOURCE = Path(os.environ.get('JPDEV_CLEAN_SOURCE', KIT / 'lib/clean.py')) +spec = importlib.util.spec_from_file_location('clean', SOURCE) +clean = importlib.util.module_from_spec(spec) +spec.loader.exec_module(clean) + + +class Cleanup(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='clean-fixture-') + self.addCleanup(self.temp.cleanup) + self.home = Path(self.temp.name) + self.root = self.home / 'session' + clean.initialize(str(self.root), 'session-test') + self.outside = self.home / 'outside' + self.outside.mkdir() + (self.outside / 'keep').write_text('operator evidence') + self.rootctx = clean.directory(str(self.root)) + self.fd = self.rootctx.__enter__() + self.addCleanup(self.rootctx.__exit__, None, None, None) + self.owner = clean.Root(str(self.root), self.fd) + self.addCleanup(os.close, self.owner.meta) + self.roots = [str(self.root)] + + def git(self, *args, cwd=None): + env = {**os.environ, 'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull, + 'GIT_AUTHOR_NAME': 'Fixture', 'GIT_AUTHOR_EMAIL': 'fixture@example.invalid', + 'GIT_COMMITTER_NAME': 'Fixture', 'GIT_COMMITTER_EMAIL': 'fixture@example.invalid'} + result = subprocess.run(['git', *args], cwd=cwd or self.root, env=env, + text=True, capture_output=True, check=True) + return result.stdout.strip() + + def candidate(self, kind='review', path='old', base=None, removed=None): + folder = self.root / path + folder.mkdir(parents=True, exist_ok=True) + if not (folder / '.git').exists(): + (folder / 'report').write_text('session-created evidence') + args = clean.parser().parse_args(['seal', '--root', str(self.root), '--session', 'session-test', + '--path', path, '--kind', kind]) + args.base, args.removed_worktree = base, removed + clean.seal(self.owner, args) + record = next(r for _, r in self.owner.records() if r['path'] == path) + return folder, record + + def remove(self, record, processes=lambda paths: None): + clean.remove(self.owner, record, self.roots, 0, processes) + + def refused(self, record, reason, processes=lambda paths: None): + with self.assertRaisesRegex((clean.Refusal, OSError), reason): + self.remove(record, processes) + self.assertTrue((self.root / record['path']).exists()) + self.assertEqual((self.outside / 'keep').read_text(), 'operator evidence') + + def repository(self, branch=True, review=False): + base = self.root / 'base' + remote = self.root / 'origin.git' + self.git('init', '--bare', str(remote)) + self.git('init', '-b', 'main', str(base)) + (base / 'file').write_text('published') + self.git('add', '.', cwd=base) + self.git('commit', '-qm', 'published fixture', cwd=base) + self.git('remote', 'add', 'origin', str(remote), cwd=base) + self.git('push', '-qu', 'origin', 'main', cwd=base) + folder = self.root / 'old' + if review: + self.git('clone', str(remote), str(folder)) + self.git('checkout', '--detach', 'origin/main', cwd=folder) + elif branch: + self.git('worktree', 'add', '-b', 'topic', str(folder), cwd=base) + self.git('push', '-qu', 'origin', 'topic', cwd=folder) + else: + self.git('worktree', 'add', '--detach', str(folder), cwd=base) + return folder, base, remote + + @contextlib.contextmanager + def pr(self, state='MERGED'): + original = clean.run + def command(args, cwd): + if args[0] == 'gh': + return json.dumps([{'state': state, 'headRefName': 'topic', + 'headRefOid': self.git('rev-parse', 'HEAD', cwd=cwd)}]) + return original(args, cwd) + with patch.object(clean, 'run', command): + yield + + def test_safe_classes(self): + for kind in ('review', 'transcript'): + with self.subTest(kind=kind): + folder, record = self.candidate(kind, path=kind) + self.remove(record) + self.assertFalse(folder.exists()) + folder, base, _ = self.repository(branch=False) + _, record = self.candidate('worktree') + self.remove(record) + self.assertFalse(folder.exists()) + # Stale Git admin entries are harmless; live worktree directories determine cache use. + for kind in ('scratch', 'cache'): + candidate, record = self.candidate(kind, path=kind, base=str(base), removed=str(folder)) + self.remove(record) + self.assertFalse(candidate.exists()) + + def test_closed_and_merged_pr(self): + self.repository() + _, record = self.candidate('worktree') + with self.pr('CLOSED'): + clean.inspect(self.owner, record, self.roots, 0, lambda paths: None) + with self.pr(): + self.remove(record) + self.assertFalse((self.root / 'old').exists()) + + def test_review_clone(self): + self.repository(review=True) + folder, record = self.candidate('review') + self.remove(record) + self.assertFalse(folder.exists()) + + def test_attached_review_clone(self): + folder, _, _ = self.repository(review=True) + self.git('checkout', '-b', 'topic', '--track', 'origin/main', cwd=folder) + _, record = self.candidate('review') + with self.pr(): + self.remove(record) + self.assertFalse(folder.exists()) + + def test_review_base_with_worktrees(self): + _, base, _ = self.repository(branch=False) + self.git('checkout', '--detach', cwd=base) + _, record = self.candidate('review', path='base') + self.refused(record, 'base still has registered worktrees') + + def test_dry_run_and_absent_root(self): + folder, record = self.candidate() + with patch.object(clean, 'process_check', lambda paths: None), contextlib.redirect_stdout(io.StringIO()) as output: + # Defaults bind at definition, so replace only the live evidence seam here. + with patch.object(clean, 'inspect', lambda *a: (record['inventory'], None)): + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s']), 0) + self.assertTrue(folder.exists()) + self.assertIn('"bytes":', output.getvalue()) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(clean.main([]), 0) + self.assertEqual(clean.main(['--root', str(self.home / 'absent'), '--yes']), 0) + + def test_age(self): + _, record = self.candidate() + with self.assertRaisesRegex(clean.Refusal, 'younger'): + clean.remove(self.owner, record, self.roots, 86400, lambda paths: None) + + def test_unmarked_and_other_session(self): + unmarked = self.home / 'unmarked' + unmarked.mkdir() + (unmarked / 'historical').write_text('not disposable') + with self.assertRaisesRegex(clean.Refusal, 'empty'): + clean.initialize(str(unmarked), 'session-test') + _, record = self.candidate() + record['session'] = 'another-session' + clean.write_json(self.owner.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json', record) + self.refused(record, 'ownership') + + def test_unknown_ignored_untracked_content(self): + self.repository(branch=False) + folder, record = self.candidate('worktree') + self.git('config', 'core.excludesFile', str(folder / 'ignore'), cwd=folder) + for filename in ('untracked', 'ignored'): + with self.subTest(filename=filename): + (folder / 'ignore').write_text('ignored\n') + (folder / filename).write_text('new evidence') + self.refused(record, 'disposal evidence') + (folder / filename).unlink() + (folder / 'ignore').unlink() + # Explicit disposal seal allows the known untracked/ignored evidence. + (folder / 'untracked').write_text('explicitly disposable') + (folder / '.gitignore').write_text('ignored\n') + (folder / 'ignored').write_text('explicitly disposable') + _, record = self.candidate('worktree') + self.remove(record) + self.assertFalse(folder.exists()) + + def test_tracked_change(self): + folder, _, _ = self.repository(branch=False) + (folder / 'file').write_text('changed') + _, record = self.candidate('worktree') + self.refused(record, 'tracked changes') + + def test_unpublished_other_ref(self): + folder, base, _ = self.repository(branch=False) + self.git('checkout', '-b', 'private', cwd=base) + (base / 'secret').write_text('unpublished') + self.git('add', '.', cwd=base) + self.git('commit', '-qm', 'private fixture', cwd=base) + # HEAD in the candidate is published; the unpublished commit survives only in another ref. + _, record = self.candidate('worktree') + self.refused(record, 'unpublished') + self.assertTrue(folder.exists()) + + def test_missing_upstream_and_open_pr(self): + folder, _, _ = self.repository() + self.git('branch', '--unset-upstream', cwd=folder) + _, record = self.candidate('worktree') + with self.pr(): + self.refused(record, 'upstream') + self.git('branch', '--set-upstream-to=origin/topic', cwd=folder) + with self.pr('OPEN'): + self.refused(record, 'PR is not verified') + + def test_unavailable_git_remote_pr(self): + self.repository() + _, record = self.candidate('worktree') + original = clean.run + for target in ('git', 'ls-remote', 'gh'): + def command(args, cwd): + if args[0] == target or target in args: + raise clean.Refusal('verification unavailable fixture') + if args[0] == 'gh': + return '[]' + return original(args, cwd) + with self.subTest(target=target), patch.object(clean, 'run', command): + self.refused(record, 'verification unavailable') + + def test_keep_and_protected_parent(self): + _, record = self.candidate(path='parent/old') + for keep in ('parent', 'parent/old/report'): + (self.root / '.jpdev-clean-keep').write_text(keep + '\n') + self.refused(record, 'operator keep') + (self.root / '.jpdev-clean-keep').unlink() + self.roots.append(str(self.root / 'parent/old/protected')) + self.refused(record, 'protected') + + def test_live_checkout(self): + folder, record = self.candidate() + with patch.object(clean.os, 'getcwd', lambda: str(folder)): + self.refused(record, 'protected') + + def test_symlink(self): + folder, _ = self.candidate() + (folder / 'link').symlink_to(self.outside, target_is_directory=True) + with self.assertRaisesRegex(clean.Refusal, 'symlink'): + self.candidate() + # A sealed path replaced by a link cannot redirect removal. + (folder / 'link').unlink() + _, record = self.candidate() + folder.rename(self.root / 'saved') + folder.symlink_to(self.outside, target_is_directory=True) + with self.assertRaises((clean.Refusal, OSError)): + self.remove(record) + self.assertEqual((self.outside / 'keep').read_text(), 'operator evidence') + + def test_escape(self): + for path in ('../outside', '/outside', 'old/../outside', '.jpdev-clean', 'old//x', ''): + with self.subTest(path=path), self.assertRaises(clean.Refusal): + clean.relative(path) + + def test_nested_worktree(self): + folder, base, _ = self.repository(branch=False) + _, record = self.candidate('worktree') + self.git('worktree', 'add', '--detach', str(folder / 'nested'), cwd=base) + expected = dict(record['inventory']) + for path in sorted((folder / 'nested').rglob('*')) + [folder / 'nested']: + st = path.lstat() + value = [st.st_dev, st.st_ino, st.st_mode, st.st_size, st.st_mtime_ns, st.st_ctime_ns] + if stat.S_ISREG(st.st_mode): + value.append(hashlib.sha256(path.read_bytes()).hexdigest()) + expected[str(path.relative_to(folder))] = value + # This hand-built seal inventory attests even the nested fixture, which remains protected. + record['inventory'] = expected + clean.write_json(self.owner.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json', record) + self.refused(record, 'nested repository') + self.assertTrue((folder / 'nested/file').exists()) + + def test_root_replaced(self): + _, record = self.candidate() + self.root.rename(self.home / 'saved') + self.root.mkdir() + (self.home / 'saved' / clean.META).rename(self.root / clean.META) + (self.root / 'old').mkdir() + (self.root / 'old/report').write_text('replacement evidence') + with self.assertRaisesRegex(clean.Refusal, 'root replaced'): + self.remove(record) + self.assertTrue((self.root / 'old/report').exists()) + self.assertTrue((self.home / 'saved/old/report').exists()) + + def test_candidate_replaced(self): + folder, _ = self.candidate() + (folder / 'report').unlink() + args = clean.parser().parse_args(['seal', '--root', str(self.root), '--session', 'session-test', + '--path', 'old', '--kind', 'review']) + clean.seal(self.owner, args) + record = next(r for _, r in self.owner.records()) + folder.rename(self.root / 'saved') + folder.mkdir() + self.refused(record, 'candidate replaced') + + def test_changed_during_quarantine(self): + folder, record = self.candidate() + rename = clean.os.rename + def changed(src, dst, **kwargs): + rename(src, dst, **kwargs) + if dst.startswith('.jpdev-clean-removing-'): + with open(self.root / dst / 'report', 'w') as stream: + stream.write('concurrent evidence') + with patch.object(clean.os, 'rename', changed): + self.refused(record, 'content changed during quarantine') + self.assertEqual((folder / 'report').read_text(), 'concurrent evidence') + + def test_live_pid_and_unknown_process_evidence(self): + folder, record = self.candidate() + proc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=folder) + self.addCleanup(proc.wait) + self.addCleanup(proc.terminate) + # Keep real /proc data while substituting only initial-namespace visibility evidence. + readlink = os.readlink + def host(path, *args, **kwargs): + if path in ('/proc/self/ns/pid', '/proc/1/ns/pid'): + return 'pid:[4026531836]' + if path == '/proc/self/ns/mnt': + return readlink('/proc/1/ns/mnt') + return readlink(path, *args, **kwargs) + with patch.object(clean.os, 'readlink', host): + self.refused(record, 'live PID ' + str(proc.pid), clean.process_check) + with patch.object(clean.os, 'readlink', side_effect=PermissionError('fixture')): + self.refused(record, 'process visibility unavailable', clean.process_check) + with patch.object(clean.os, 'readlink', return_value='pid:[nested]'): + self.refused(record, 'process visibility incomplete', clean.process_check) + + def test_cache_live_users_and_scratch_owner(self): + folder, base, _ = self.repository(branch=False) + _, record = self.candidate('cache', path='cache', base=str(base)) + self.refused(record, 'worktree users') + _, record = self.candidate('scratch', path='scratch', base=str(base), removed=str(folder)) + self.refused(record, 'not removed') + self.git('init', '-b', 'main', str(self.outside)) + _, record = self.candidate('cache', path='bad-cache', base=str(self.outside)) + self.refused(record, 'outside authorized roots') + + def test_transcript_file(self): + log = self.root / 'transcript.log' + log.write_text('session transcript') + args = clean.parser().parse_args(['seal', '--root', str(self.root), '--session', 'session-test', + '--path', 'transcript.log', '--kind', 'transcript']) + clean.seal(self.owner, args) + record = next(r for _, r in self.owner.records()) + self.remove(record) + self.assertFalse(log.exists()) + + def test_main_removal_and_error_status(self): + folder, record = self.candidate() + inspect, remove = clean.inspect, clean.remove + def inspect_fixture(*args): + return inspect(*args[:4], processes=lambda paths: None) + def remove_fixture(*args): + return remove(*args, processes=lambda paths: None) + with patch.object(clean, 'inspect', inspect_fixture), patch.object(clean, 'remove', remove_fixture): + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', '--yes']), 0) + self.assertFalse(folder.exists()) + self.assertIn('"outcome": "removed"', output.getvalue()) + self.assertIn('"removed": 1', output.getvalue()) + folder, record = self.candidate() + with patch.object(clean.os, 'unlink', side_effect=PermissionError('fixture denied')): + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', '--yes']), 1) + self.assertTrue(folder.exists()) + self.assertIn('fixture denied', output.getvalue()) + + def test_parent_symlink_after_inspection(self): + folder, record = self.candidate(path='parent/old') + original = clean.inspect + def replace(*args, **kwargs): + result = original(*args, **kwargs) + (self.root / 'parent').rename(self.root / 'saved-parent') + (self.root / 'parent').symlink_to(self.outside, target_is_directory=True) + return result + with patch.object(clean, 'inspect', replace): + with self.assertRaises(OSError): + self.remove(record) + self.assertTrue((self.root / 'saved-parent/old/report').exists()) + self.assertTrue((self.outside / 'keep').exists()) + + def test_changed_while_erasing(self): + folder, record = self.candidate() + original = clean.erase + def replace(fd, expected, prefix='', before_unlink=None): + os.rename('report', str(self.root / 'rescued-report'), src_dir_fd=fd) + return original(fd, expected, prefix, before_unlink) + with patch.object(clean, 'erase', replace): + self.refused(record, 'directory changed during removal') + self.assertTrue((self.root / 'rescued-report').exists()) + self.assertTrue(folder.exists()) + + def test_process_starts_during_quarantine(self): + folder, record = self.candidate() + calls = [] + def processes(paths): + calls.append(paths) + if any('.jpdev-clean-removing-' in path for path in paths): + raise clean.Refusal('live PID fixture-after-inspection') + self.refused(record, 'live PID fixture-after-inspection', processes) + self.assertGreaterEqual(len(calls), 2) + self.assertTrue((folder / 'report').exists()) + + def test_git_changes_during_quarantine(self): + folder, base, _ = self.repository(branch=False) + _, record = self.candidate('worktree') + rename = clean.os.rename + def changed(src, dst, **kwargs): + rename(src, dst, **kwargs) + if dst.startswith('.jpdev-clean-removing-'): + self.git('checkout', '-b', 'private', cwd=base) + (base / 'private').write_text('concurrent private commit') + self.git('add', '.', cwd=base) + self.git('commit', '-qm', 'concurrent fixture', cwd=base) + with patch.object(clean.os, 'rename', changed): + self.refused(record, 'unpublished') + self.assertTrue((folder / 'file').exists()) + + def test_root_and_parent_symlink(self): + folder, record = self.candidate() + link = self.home / 'linked-root' + link.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(OSError): + with clean.directory(str(link)) as fd: + alias = clean.Root(str(link), fd) + try: + clean.remove(alias, record, [str(link)], 0, lambda paths: None) + finally: + os.close(alias.meta) + self.assertTrue((folder / 'report').exists()) + + def test_invalid_duration_and_inventory(self): + for text in ('', '-1d', '2', '1.5h', 'forever'): + with self.subTest(text=text), self.assertRaises(Exception): + clean.duration(text) + self.assertEqual(clean.duration('2h'), 7200) + folder, _ = self.candidate() + os.mkfifo(str(folder / 'fifo')) + with self.assertRaisesRegex(clean.Refusal, 'special file'): + self.candidate() + + def test_staged_change(self): + folder, _, _ = self.repository(branch=False) + (folder / 'file').write_text('staged change') + self.git('add', 'file', cwd=folder) + (folder / 'file').write_text('published') + _, record = self.candidate('worktree') + self.refused(record, 'tracked') + + def test_upstream_not_containing_head(self): + folder, base, _ = self.repository() + (folder / 'file').write_text('new published commit') + self.git('add', 'file', cwd=folder) + self.git('commit', '-qm', 'published topic', cwd=folder) + self.git('push', '-q', 'origin', 'topic', cwd=folder) + self.git('branch', '--set-upstream-to=origin/main', cwd=folder) + _, record = self.candidate('worktree') + with self.pr(): + self.refused(record, 'upstream does not contain') + + def test_missing_worktree_registration_outside_roots(self): + folder, base, _ = self.repository(branch=False) + absent = self.outside / 'absent-worktree' + self.git('worktree', 'move', str(folder), str(absent), cwd=base) + self.git('worktree', 'remove', str(absent), cwd=base) + # Register a worktree outside the authorized root, then remove only its fixture files. + self.git('worktree', 'add', '--detach', str(absent), cwd=base) + import shutil + shutil.rmtree(str(absent)) + _, record = self.candidate('cache', path='cache', base=str(base)) + self.refused(record, 'worktree evidence outside') + + def test_git_evidence_outside_roots(self): + folder, base, _ = self.repository(branch=False) + moved = self.outside / 'base' + self.git('worktree', 'move', str(folder), str(self.root / 'moving'), cwd=base) + # Narrow authorization to just the worktree's parent; Git metadata remains outside it. + self.roots = [str(self.root / 'moving')] + (self.root / 'moving').rename(folder) + _, record = self.candidate('worktree') + self.refused(record, 'outside authorized roots') + self.assertFalse(moved.exists()) + + def test_root_permissions_changed(self): + _, record = self.candidate() + self.root.chmod(0o777) + self.addCleanup(self.root.chmod, 0o700) + self.refused(record, 'root ownership or permissions changed') + + def test_disposal_metadata_changed(self): + _, record = self.candidate() + (self.root / clean.META).rename(self.root / 'saved-metadata') + (self.root / clean.META).mkdir() + self.refused(record, 'ownership directory replaced') + + def test_parent_replaced_during_quarantine(self): + _, record = self.candidate(path='parent/old') + rename = clean.os.rename + def changed(src, dst, **kwargs): + rename(src, dst, **kwargs) + if dst.startswith('.jpdev-clean-removing-'): + (self.root / 'parent').rename(self.root / 'saved-parent') + (self.root / 'parent').mkdir() + with patch.object(clean.os, 'rename', changed): + with self.assertRaisesRegex(clean.Refusal, 'parent replaced'): + self.remove(record) + self.assertTrue((self.root / 'saved-parent/old/report').exists()) + + def test_file_replaced_during_erase(self): + folder, record = self.candidate() + original = clean.erase + def changed(fd, expected, prefix='', before_unlink=None): + def race(): + with open('/proc/self/fd/' + str(fd) + '/report', 'w') as stream: + stream.write('new concurrent content') + return original(fd, expected, prefix, race) + with patch.object(clean, 'erase', changed): + self.refused(record, 'file replaced during removal') + self.assertEqual((folder / 'report').read_text(), 'new concurrent content') + + def test_unverified_upstream_remote(self): + folder, _, remote = self.repository() + self.git('remote', 'add', 'unverified', str(remote), cwd=folder) + self.git('config', 'branch.topic.remote', 'unverified', cwd=folder) + self.git('fetch', '-q', 'unverified', cwd=folder) + _, record = self.candidate('worktree') + with self.pr(): + self.refused(record, 'upstream publication unavailable') + + def test_special_file_cleanup(self): + folder, record = self.candidate() + fifo = folder / 'fifo' + os.mkfifo(str(fifo)) + st = fifo.lstat() + record['inventory']['fifo'] = [st.st_dev, st.st_ino, st.st_mode, st.st_size, + st.st_mtime_ns, st.st_ctime_ns, + hashlib.sha256(b'').hexdigest()] + clean.write_json(self.owner.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json', record) + self.refused(record, 'special file') + self.assertTrue(fifo.exists()) + + def test_lock_and_absent_seal(self): + folder, record = self.candidate() + inspect, remove = clean.inspect, clean.remove + clean.fcntl.flock(self.owner.meta, clean.fcntl.LOCK_EX | clean.fcntl.LOCK_NB) + with patch.object(clean, 'inspect', lambda *args: inspect(*args[:4], processes=lambda paths: None)), \ + patch.object(clean, 'remove', lambda *args: remove(*args, processes=lambda paths: None)): + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', '--yes']), 1) + self.assertTrue(folder.exists()) + with self.assertRaises(clean.Refusal), contextlib.redirect_stdout(io.StringIO()): + clean.main(['seal', '--root', str(self.home / 'absent'), '--session', 'session-test', + '--path', 'old', '--kind', 'review']) + + def test_host_mount_visibility(self): + _, record = self.candidate() + original = os.readlink + def isolated(path, *args, **kwargs): + if path in ('/proc/self/ns/pid', '/proc/1/ns/pid'): + return 'pid:[4026531836]' + if path == '/proc/self/ns/mnt': + return 'mnt:[isolated]' + return original(path, *args, **kwargs) + def processes(paths): + with patch.object(clean.os, 'readlink', isolated), patch.object(clean.os, 'listdir', return_value=[]): + clean.process_check(paths) + self.refused(record, 'host mount/root view required', processes) + + def test_parent_escape_removal(self): + st = (self.outside / 'keep').lstat() + record = {'version': 1, 'session': 'session-test', 'path': '../outside', 'kind': 'review', + 'identity': [self.outside.stat().st_dev, self.outside.stat().st_ino], 'sealed': 0, + 'inventory': {'keep': [st.st_dev, st.st_ino, st.st_mode, st.st_size, + st.st_mtime_ns, st.st_ctime_ns, + hashlib.sha256(b'operator evidence').hexdigest()]}} + clean.write_json(self.owner.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json', record) + with self.assertRaisesRegex(clean.Refusal, 'invalid or protected relative path'): + self.remove(record) + self.assertTrue((self.outside / 'keep').exists()) + + def test_base_checkout(self): + folder, base, _ = self.repository(branch=False) + self.git('worktree', 'remove', str(folder), cwd=base) + self.git('checkout', '--detach', cwd=base) + _, record = self.candidate('worktree', path='base') + self.refused(record, 'live base checkout is protected') + self.assertTrue((base / 'file').exists()) + + def test_changed_seal_during_quarantine(self): + folder, record = self.candidate() + rename = clean.os.rename + def changed(src, dst, **kwargs): + rename(src, dst, **kwargs) + if dst.startswith('.jpdev-clean-removing-'): + replacement = dict(record) + replacement['sealed'] += 1 + clean.write_json(self.owner.meta, hashlib.sha256(record['path'].encode()).hexdigest() + '.json', replacement) + with patch.object(clean.os, 'rename', changed): + self.refused(record, 'disposal seal changed') + self.assertTrue((folder / 'report').exists()) + + def test_changed_metadata_permissions(self): + _, record = self.candidate() + metadata = self.root / clean.META + metadata.chmod(0o777) + self.addCleanup(metadata.chmod, 0o700) + self.refused(record, 'ownership directory permissions changed') + + def test_git_changes_during_erase(self): + folder, base, _ = self.repository(branch=False) + _, record = self.candidate('worktree') + original = clean.erase + def changed(fd, expected, prefix='', before_unlink=None): + self.git('checkout', '-b', 'private', cwd=base) + (base / 'secret').write_text('new private evidence') + self.git('add', '.', cwd=base) + self.git('commit', '-qm', 'concurrent fixture', cwd=base) + return original(fd, expected, prefix, before_unlink) + with patch.object(clean, 'erase', changed): + self.refused(record, 'Git state changed during removal') + self.assertTrue((folder / 'file').exists()) + + def test_mount_crossing(self): + folder, record = self.candidate() + original = open + def mounts(path, *args, **kwargs): + if path == '/proc/self/mountinfo': + return io.StringIO('20 1 0:1 / ' + str(folder / 'report') + ' rw - fixture fixture rw\n') + return original(path, *args, **kwargs) + with patch('builtins.open', mounts): + self.refused(record, 'mounted content') + self.assertTrue((folder / 'report').exists()) + + def test_disappearing_ownership_record(self): + folder, _ = self.candidate() + original = clean.Root.records + def disappear(root): + for name, record in original(root): + raise FileNotFoundError('concurrent ownership evidence removal') + yield name, record + with patch.object(clean.Root, 'records', disappear), contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(clean.main(['--root', str(self.root), '--yes']), 1) + self.assertIn('ownership evidence disappeared', output.getvalue()) + self.assertTrue((folder / 'report').exists()) + + def test_candidate_ownership_changed(self): + _, record = self.candidate() + original = clean.os.stat + def other_owner(path, *args, **kwargs): + st = original(path, *args, **kwargs) + if path == 'old': + fields = {name: getattr(st, name) for name in dir(st) if name.startswith('st_')} + fields['st_uid'] = os.getuid() + 1 + return SimpleNamespace(**fields) + return st + with patch.object(clean.os, 'stat', other_owner): + self.refused(record, 'candidate ownership changed') + + def test_unlink_failure(self): + _, record = self.candidate() + with patch.object(clean.os, 'unlink', side_effect=PermissionError('removal denied')): + self.refused(record, 'removal denied') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run.sh b/tests/run.sh index aab98f8..28ca23c 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -7,6 +7,9 @@ work=$(mktemp -d) trap 'rm -rf -- "${work:?}"' EXIT export JPDEV_SCRATCH=$work/scratch XDG_CONFIG_HOME=$work/config HOME=$work/home mkdir -p "$HOME" "$XDG_CONFIG_HOME" +# A clone-local TMPDIR must not make an outside-repository fixture discover the kit. +GIT_CEILING_DIRECTORIES="$(dirname "$work")${GIT_CEILING_DIRECTORIES:+:$GIT_CEILING_DIRECTORIES}" +export GIT_CEILING_DIRECTORIES unset CI JPDEV_VENV JPDEV_LOCK JPDEV_NODE_RSS_MB JPDEV_NODE_TIMEOUT_S JPDEV_GO_ULIMIT_KB JPDEV_GO_TIMEOUT_S JPDEV_PY_ULIMIT_KB JPDEV_NVM_DIR JPDEV_GO_SDK_DIR JPDEV_NAME_PATTERNS JPDEV_CODEX_MODEL_DEFAULT JPDEV_CODEX_SANDBOX JPDEV_CODEX_WORDS JPDEV_FIXTURE_UNSET # What jpdev check exports to its steps: a run of these tests under it must not inherit it. unset JPDEV_ROOT JPDEV_WRAP JPDEV_NODE_BIN JPDEV_GO_BIN JPDEV_LOGDIR JPDEV_STEP_TIMEOUT @@ -538,6 +541,22 @@ if [ -e "$kit/tests/fixture/.mutated" ]; then bad "jpdev mutants leaves the clon if [ -e "$kit/.mutated" ]; then bad "jpdev mutants ran in the clone itself"; rm -f "$kit/.mutated"; else ok "jpdev mutants ran in a scratch copy"; fi if ls "$JPDEV_SCRATCH"/mutants/* > /dev/null 2>&1; then bad "jpdev mutants removes its scratch copy"; else ok "jpdev mutants removes its scratch copy"; fi +# Explicit clone-local scratch/TMPDIR fixture: source siblings survive; generated trees do not. +nested_mutants=$work/nested-mutants +mkdir -p "$nested_mutants/state/tmp/node_modules" "$nested_mutants/state/scratch" +printf 'source sibling\n' > "$nested_mutants/state/keep" +git init -q -b main "$nested_mutants" +cat > "$nested_mutants/dev.json" <<'JSON' +{"version":1,"checks":[],"mutants":{"run":"bash probe.sh {rows}","toolchain":"shell"}} +JSON +cat > "$nested_mutants/probe.sh" <<'SH' +#!/usr/bin/env bash +[ -f state/keep ] && [ ! -e state/tmp ] && [ ! -e state/scratch ] || exit 6 +printf '| nested scratch | fixture | caught: generated paths excluded |\n' +SH +out=$(cd "$nested_mutants" && TMPDIR=$nested_mutants/state/tmp JPDEV_SCRATCH=$nested_mutants/state/scratch "$jpdev" mutants --row 'nested scratch' 2>&1) && s=0 || s=$? +expect "mutants excludes nested scratch and TMPDIR while keeping source siblings" 0 'caught +nested scratch' "$out" "$s" + # --- the record gate mkdir -p "$work/home" "$XDG_CONFIG_HOME/jpdev" printf '%s\n' 'unlikely-name-xyz' > "$XDG_CONFIG_HOME/jpdev/name-patterns" @@ -657,7 +676,8 @@ cat > "$work/codexbin/codex" <<'STUB' printf '%s\0' "$@" > "$STUB_CODEX_ARGS" pwd > "$STUB_CODEX_CWD" printf '%s\n' "JPDEV_SCRATCH=$JPDEV_SCRATCH" "JPDEV_LOCK=$JPDEV_LOCK" "GOCACHE=${GOCACHE:-}" \ - "GOFLAGS=${GOFLAGS:-}" "npm_config_cache=${npm_config_cache:-}" "XDG_CACHE_HOME=${XDG_CACHE_HOME:-}" > "$STUB_CODEX_ENV" + "GOFLAGS=${GOFLAGS:-}" "npm_config_cache=${npm_config_cache:-}" "XDG_CACHE_HOME=${XDG_CACHE_HOME:-}" \ + "TMPDIR=${TMPDIR:-}" > "$STUB_CODEX_ENV" timeout 2 cat > "$STUB_CODEX_STDIN" while [ $# -gt 0 ]; do case $1 in -o) printf 'stub final message\n' > "$2" ;; esac @@ -750,6 +770,23 @@ out=$("$jpdev" codex run --brief "$work/brief.md" --model m --effort low --dir " expect "codex run refuses a directory with no AGENTS.md" 2 'holds no AGENTS.md' "$out" "$s" if [ -e "$STUB_CODEX_ARGS" ]; then bad "a refused run did not start codex"; else ok "a refused run did not start codex"; fi +mkdir -p "$work/inherited-outside-temp" +out=$(TMPDIR=$work/inherited-outside-temp "$jpdev" codex run --no-commit --brief "$work/brief.md" --model model-x --effort low --dir "$work/codexclone" --log "$work/codex-tmpdir.log" 2>&1) && s=0 || s=$? +expect "codex run sets builder TMPDIR beneath scratch despite an inherited value" 0 "codex exit 0" "$out" "$s" +if grep -qxF "TMPDIR=$JPDEV_SCRATCH/tmp" "$STUB_CODEX_ENV" && [ -d "$JPDEV_SCRATCH/tmp" ]; then ok "builder TMPDIR is created beneath scratch"; else bad "builder TMPDIR is created beneath scratch" "$(cat "$STUB_CODEX_ENV")"; fi +out=$("$jpdev" clean 2>&1) && s=0 || s=$? +expect "clean without roots deletes nothing" 0 '"candidates": 0.*"removed": 0' "$out" "$s" +out=$("$jpdev" clean --root "$work/absent-clean-root" --yes 2>&1) && s=0 || s=$? +expect "clean safely reports an absent root" 0 '"outcome": "absent"' "$out" "$s" +out=$("$jpdev" clean init --root "$work/cleanup-session" --session session-test 2>&1) && s=0 || s=$? +expect "clean init enrolls a new empty session root" 0 'initialized session cleanup root' "$out" "$s" +mkdir "$work/cleanup-session/review" +printf 'session report\n' > "$work/cleanup-session/review/report" +out=$("$jpdev" clean seal --root "$work/cleanup-session" --session session-test --path review --kind review 2>&1) && s=0 || s=$? +expect "clean seal records explicit session disposal evidence" 0 'sealed explicit session disposal inventory' "$out" "$s" +out=$(python3 -I "$kit/tests/clean_test.py" 2>&1) && s=0 || s=$? +expect "clean deletion boundaries hold in disposable fixtures" 0 '^OK$' "$out" "$s" + # worktrees: creation, linked dependencies, shared caches and guarded removal wtorigin=$work/wtorigin.git wtseed=$work/wtseed From 781814ba91977e34f744b68a840909f77cc3a1fb Mon Sep 17 00:00:00 2001 From: kikashy Date: Fri, 23 Oct 2026 08:07:31 -0400 Subject: [PATCH 2/4] Make cleanup process fixtures portable across host permissions Use a deterministic host process view around the real disposable child PID so cleanup fixtures do not require access to protected PID 1 information. Keep production cleanup checks unchanged and retain mutation coverage for the live-process and host-mount safeguards. Collect failed test cases in one summary line and select it in dev.json so late failures remain visible when CI truncates raw test output. Co-Authored-By: Codex Signed-off-by: kikashy --- dev.json | 2 +- tests/clean_test.py | 19 ++++++++++++++----- tests/run.sh | 13 +++++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dev.json b/dev.json index 74e23ed..e2f4020 100644 --- a/dev.json +++ b/dev.json @@ -8,6 +8,6 @@ {"name": "bash-n", "run": "set -eu; for file in bin/* lib/*.sh templates/*.sh scripts/*.sh scripts/dev/*.sh tests/*.sh tests/fixture/*.sh; do bash -n \"$file\"; done; echo 'bash -n: every script parses'", "toolchain": "shell", "summary": "^bash -n:"}, {"name": "json", "run": "python3 -I -c 'import glob, json; files = sorted(glob.glob(\"*.json\") + glob.glob(\"templates/*.json\") + glob.glob(\"examples/*.json\") + glob.glob(\"tests/fixture/*.json\")); [json.load(open(f, encoding=\"utf-8\")) for f in files]; print(\"parsed %d JSON documents\" % len(files))'", "toolchain": "shell", "summary": "^parsed "}, {"name": "shellcheck", "run": "shellcheck -x -P \".:SCRIPTDIR\" bin/* lib/*.sh templates/*.sh scripts/*.sh scripts/dev/*.sh tests/*.sh tests/fixture/*.sh && echo 'shellcheck: clean'", "toolchain": "shell", "summary": "^shellcheck:"}, - {"name": "tests", "run": "tests/run.sh", "toolchain": "shell", "summary": "^[0-9]+ passed, [0-9]+ failed", "timeout": 900} + {"name": "tests", "run": "tests/run.sh", "toolchain": "shell", "summary": "^(Failed cases:|[0-9]+ passed, [0-9]+ failed)", "timeout": 900} ] } diff --git a/tests/clean_test.py b/tests/clean_test.py index 99b3161..c86199a 100644 --- a/tests/clean_test.py +++ b/tests/clean_test.py @@ -323,15 +323,22 @@ def test_live_pid_and_unknown_process_evidence(self): proc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=folder) self.addCleanup(proc.wait) self.addCleanup(proc.terminate) - # Keep real /proc data while substituting only initial-namespace visibility evidence. - readlink = os.readlink + # Supply a complete fixture host view without reading protected PID 1 or unrelated + # processes. The child PID and its cwd, descriptors and mappings remain real evidence. + readlink, stat_path, listdir = os.readlink, os.stat, os.listdir + mount = readlink('/proc/self/ns/mnt') def host(path, *args, **kwargs): if path in ('/proc/self/ns/pid', '/proc/1/ns/pid'): return 'pid:[4026531836]' - if path == '/proc/self/ns/mnt': - return readlink('/proc/1/ns/mnt') + if path in ('/proc/self/ns/mnt', '/proc/1/ns/mnt'): + return mount return readlink(path, *args, **kwargs) - with patch.object(clean.os, 'readlink', host): + def host_stat(path, *args, **kwargs): + return stat_path('/' if path == '/proc/1/root' else path, *args, **kwargs) + def host_pids(path): + return [str(proc.pid)] if path == '/proc' else listdir(path) + with patch.object(clean.os, 'readlink', host), patch.object(clean.os, 'stat', host_stat), \ + patch.object(clean.os, 'listdir', host_pids): self.refused(record, 'live PID ' + str(proc.pid), clean.process_check) with patch.object(clean.os, 'readlink', side_effect=PermissionError('fixture')): self.refused(record, 'process visibility unavailable', clean.process_check) @@ -573,6 +580,8 @@ def isolated(path, *args, **kwargs): return 'pid:[4026531836]' if path == '/proc/self/ns/mnt': return 'mnt:[isolated]' + if path == '/proc/1/ns/mnt': + return 'mnt:[fixture-host]' return original(path, *args, **kwargs) def processes(paths): with patch.object(clean.os, 'readlink', isolated), patch.object(clean.os, 'listdir', return_value=[]): diff --git a/tests/run.sh b/tests/run.sh index 28ca23c..be6410a 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -18,9 +18,18 @@ fixture=$kit/tests/fixture/dev.json system_python=$(python3 -c 'import sys; print(sys.executable)') system_git=$(command -v git) pass=0 fail=0 +failed_cases=() ok() { pass=$((pass + 1)); printf 'ok %s\n' "$1"; } -bad() { fail=$((fail + 1)); printf 'FAIL %s\n' "$1"; [ -z "${2:-}" ] || printf '%s\n' "$2" | head -n 20 | sed 's/^/ /'; } +bad() { fail=$((fail + 1)); failed_cases+=("$1"); printf 'FAIL %s\n' "$1"; [ -z "${2:-}" ] || printf '%s\n' "$2" | head -n 20 | sed 's/^/ /'; } +report_results() { + if [ "$fail" -ne 0 ]; then + printf 'Failed cases:' + printf ' <%s>' "${failed_cases[@]}" + printf '\n' + fi + printf '\n%d passed, %d failed\n' "$pass" "$fail" +} # expect NAME STATUS PATTERN OUTPUT STATUS-GOT expect() { local name=$1 want=$2 pattern=$3 out=$4 got=$5 @@ -1312,5 +1321,5 @@ if [ ! -e "$work/setup-outside" ] && [ ! -e "$work/clone/ran" ]; then ok "setup out=$("$jpdev" setup "$work/not-a-clone" 2>&1) && s=0 || s=$? expect "setup asks for --repo when the directory is not a clone" 2 'give --repo' "$out" "$s" -printf '\n%d passed, %d failed\n' "$pass" "$fail" +report_results [ "$fail" -eq 0 ] From 2a15dce41e3ff5dfaf54b896014264887c53b52e Mon Sep 17 00:00:00 2001 From: kikashy Date: Fri, 23 Oct 2026 08:24:39 -0400 Subject: [PATCH 3/4] Use schema-valid fixtures and handle PathLike rename targets Give the nested-mutants fixture a nonempty check list so it validates when jsonschema is installed. Normalize the parent-replacement rename target with os.fspath so the fixture accepts pathlib arguments on newer Python. Keep production safeguards unchanged. Verify both failing CI cases from clean archived source and retain the parent-identity safeguard mutation. Co-Authored-By: Codex Signed-off-by: kikashy --- tests/clean_test.py | 2 +- tests/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clean_test.py b/tests/clean_test.py index c86199a..3b620f1 100644 --- a/tests/clean_test.py +++ b/tests/clean_test.py @@ -518,7 +518,7 @@ def test_parent_replaced_during_quarantine(self): rename = clean.os.rename def changed(src, dst, **kwargs): rename(src, dst, **kwargs) - if dst.startswith('.jpdev-clean-removing-'): + if os.fspath(dst).startswith('.jpdev-clean-removing-'): (self.root / 'parent').rename(self.root / 'saved-parent') (self.root / 'parent').mkdir() with patch.object(clean.os, 'rename', changed): diff --git a/tests/run.sh b/tests/run.sh index be6410a..b008f1f 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -556,7 +556,7 @@ mkdir -p "$nested_mutants/state/tmp/node_modules" "$nested_mutants/state/scratch printf 'source sibling\n' > "$nested_mutants/state/keep" git init -q -b main "$nested_mutants" cat > "$nested_mutants/dev.json" <<'JSON' -{"version":1,"checks":[],"mutants":{"run":"bash probe.sh {rows}","toolchain":"shell"}} +{"version":1,"checks":[{"name":"fixture","run":"true"}],"mutants":{"run":"bash probe.sh {rows}","toolchain":"shell"}} JSON cat > "$nested_mutants/probe.sh" <<'SH' #!/usr/bin/env bash From eea02d969338de33605d700f26b5e78b1dd4ec81 Mon Sep 17 00:00:00 2001 From: kikashy Date: Fri, 23 Oct 2026 08:58:32 -0400 Subject: [PATCH 4/4] Refuse redirected Git evidence and recheck cleanup eligibility Protect bare and saved Git directories at every candidate depth. Refuse inherited Git evidence settings and index flags that can hide tracked changes, and isolate verification from ambient Git configuration. Repeat cache and scratch worktree eligibility after quarantine and before each removal. Add a transcript-only logical byte floor with a 10 MiB default and an explicit nonnegative override. Convert the independent review reproductions into refusal regressions. Verify the affected safeguards with focused tests and caught mutations. Co-Authored-By: Codex Signed-off-by: kikashy --- README.md | 5 +- bin/jpdev | 3 +- docs/policy.md | 27 ++++-- lib/clean.py | 128 ++++++++++++++++++--------- tests/clean_test.py | 204 ++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 310 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 4352be5..1abc6d3 100644 --- a/README.md +++ b/README.md @@ -107,12 +107,15 @@ the same name overrides it. `config/env.example` documents each setting with its `git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin BRANCH`, so the push uses the `gh` login and no stored credential. It refuses `--force`, `-f` and a `+` refspec. -- `jpdev clean [--root DIR]... [--older-than DURATION] [--yes]`: dry-run cleanup of explicit +- `jpdev clean [--root DIR]... [--older-than DURATION] [--min-transcript-bytes N] [--yes]`: dry-run cleanup of explicit session disposal seals; no roots means no deletion. Enroll an empty session root with `jpdev clean init --root DIR --session TOKEN`, then attest completed disposable paths with `jpdev clean seal --root DIR --session TOKEN --path REL --kind KIND` (worktree, review, scratch, transcript or cache). Scratch also needs `--base BASE --removed-worktree OLD`; shared caches need `--base BASE`. The default age is seven days (`7d`); durations use integer s/m/h/d/w units. + Transcripts require at least 10485760 logical regular-file bytes (10 MiB); use + `--min-transcript-bytes N` to change this inclusive, transcript-only floor, or `0` to disable it. + Bare or saved Git directories and inherited Git evidence redirection are refused. `.jpdev-clean-keep` holds exact relative reserves. JSON Lines reports candidates, sizes, reasons, outcomes and totals; refusals exit nonzero. Linux host process visibility, disposal inventory, Git/remote/PR evidence and all deletion checks remain mandatory with `--yes`. See diff --git a/bin/jpdev b/bin/jpdev index 28bc2c0..c2dae49 100755 --- a/bin/jpdev +++ b/bin/jpdev @@ -22,8 +22,9 @@ usage: jpdev COMMAND [ARGS] name-pattern file, then post the file push [BRANCH] push BRANCH (default: the current branch) to origin with the gh credential helper and no stored credential; refuses --force and -f - clean [--root DIR]... [--older-than DURATION] [--yes] + clean [--root DIR]... [--older-than DURATION] [--min-transcript-bytes N] [--yes] inspect explicit session disposal seals; dry run by default + transcripts default to at least 10485760 logical bytes; 0 disables the floor clean init --root DIR --session TOKEN clean seal --root DIR --session TOKEN --path REL --kind KIND [--base DIR] [--removed-worktree DIR] diff --git a/docs/policy.md b/docs/policy.md index 971f798..965c717 100644 --- a/docs/policy.md +++ b/docs/policy.md @@ -174,13 +174,16 @@ registration entries for verified absent paths beneath authorized, traversable p retained and do not count as active users. Cleanup does not prune Git administrative records or remove branch refs: a broad prune could affect another session. -`jpdev clean [--root DIR]... [--older-than DURATION] [--yes]` discovers only disposal seals, +`jpdev clean [--root DIR]... [--older-than DURATION] [--min-transcript-bytes N] [--yes]` discovers only disposal seals, never recursively guesses candidates from directory names. There are no implicit roots; no flags means no deletion. Missing roots are reported safely. It defaults to a dry run and seven days; DURATION is a nonnegative integer followed by `s`, `m`, `h`, `d` or `w`. Age is measured from the newest seal timestamp, candidate directory/file mtime and inventoried content mtime. Thus a new -seal must age even if its logs are old. There is no size cutoff: sessions can seal large logs -individually. Paths are root-relative, exact, and cannot contain empty, dot or parent components. +seal must age even if its logs are old. Transcripts/logs require at least 10485760 bytes (10 MiB) +by default. `--min-transcript-bytes N` sets an inclusive, nonnegative integer floor; `0` disables +it. The floor applies only to the `transcript` kind and counts logical regular-file bytes, +summing all regular files for a directory; directory sizes do not count. Other kinds are unaffected. +Paths are root-relative, exact, and cannot contain empty, dot or parent components. Overlapping candidate seals are refused. The ownership metadata and keep file are protected. Place exact root-relative reserves in `DIR/.jpdev-clean-keep`, one per line; blank lines and @@ -198,7 +201,19 @@ fetching. Locally unavailable remote objects cause refusal. Attached branches re origin upstream containing HEAD and a GitHub PR with matching branch and HEAD in CLOSED or MERGED state. Detached worktrees and detached reviewer clones use verified remote reachability without a PR. Reviewer clones with attached branches use the same upstream and PR checks. -A reviewer clone with any registered linked worktree remains protected. Git, remote and PR +A reviewer clone with any registered linked worktree remains protected. Bare repositories and +saved Git metadata directories at the candidate root or nested within it are unsupported and +refused, including partial HEAD/index/commondir layouts. Only a verified candidate's canonical +`.git` metadata directory is allowed. Assume-unchanged and skip-worktree index flags are refused, +even with an explicit disposal seal: they can hide changed tracked contents. + +Inherited `GIT_*` settings are refused except `GIT_OPTIONAL_LOCKS`, `GIT_TERMINAL_PROMPT`, +`GIT_PAGER` and `GIT_CEILING_DIRECTORIES`. Cleanup forces noninteractive, read-only Git commands +and disables the pager; the ceiling only bounds discovery. Index, repository, object, namespace, +shallow-history, executable and config redirection are therefore refused, including unknown +Git settings. System/global Git config and replacement objects are disabled for verification; +external diff and text conversion are disabled for tracked-diff evidence. Authentication that +depends on ambient Git config can consequently be unavailable and cause refusal. Git, remote and PR evidence must remain available; an unavailable tool, network or authentication causes refusal. `gh` is needed only for attached branches. @@ -214,7 +229,9 @@ mount crossings and changed ownership are refused. The tool requires directory d Before removal the tool repeats inspection, moves the candidate to a private sibling quarantine name, and rechecks its full inventory, Git state, keeps, root and parent identities and process -activity. Removal uses directory descriptors, never recursive path-following deletion. Each +activity. Cache worktree users and scratch associated-worktree absence are rechecked after +quarantine and before each removal, including the candidate directory itself. Removal uses directory +descriptors, never recursive path-following deletion. Each entry's identity and inventory is checked during removal. External linked-worktree refs, HEAD and index are rechecked during removal as well. Changes or failures stop that candidate; remaining content is restored to its original name when vacant, otherwise retained under diff --git a/lib/clean.py b/lib/clean.py index 99e62ae..b7ecd6d 100644 --- a/lib/clean.py +++ b/lib/clean.py @@ -16,6 +16,7 @@ META = '.jpdev-clean' KINDS = ('worktree', 'review', 'scratch', 'transcript', 'cache') +MIN_TRANSCRIPT_BYTES = 10 * 1024 * 1024 FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC @@ -133,10 +134,17 @@ def inventory(fd, prefix=''): def run(args, cwd): + # Explicit --git-dir/--work-tree do not override index, object or config redirection. + allowed = {'GIT_OPTIONAL_LOCKS', 'GIT_TERMINAL_PROMPT', 'GIT_CEILING_DIRECTORIES', 'GIT_PAGER'} + redirected = sorted(key for key in os.environ if key.startswith('GIT_') and key not in allowed) + require(not redirected, 'inherited Git evidence settings: ' + ', '.join(redirected)) try: completed = subprocess.run(args, cwd=cwd, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=30, env={**os.environ, 'GIT_OPTIONAL_LOCKS': '0', - 'GIT_TERMINAL_PROMPT': '0'}) + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': os.devnull, + 'GIT_NO_REPLACE_OBJECTS': '1', 'GIT_PAGER': 'cat'}) except (OSError, subprocess.TimeoutExpired) as exc: raise Refusal('verification unavailable: ' + args[0]) from exc require(completed.returncode == 0, 'verification failed: ' + ' '.join(args[:3])) @@ -278,8 +286,11 @@ def gitrun(args): other = item[9:] require(common != gitdir or other == path, 'base still has registered worktrees') require(not (other != path and other.startswith(path + '/')), 'nested worktree: ' + other) - require(not gitrun(['diff', '--name-only', 'HEAD', '--']), 'tracked changes') - require(not gitrun(['diff', '--cached', '--name-only', '--']), 'tracked index changes') + flags = gitrun(['ls-files', '-v', '-z']).split('\0') + require(all(not item or (not item[0].islower() and item[0] != 'S') for item in flags), + 'unsupported Git index flags: assume-unchanged or skip-worktree') + require(not gitrun(['diff', '--no-ext-diff', '--no-textconv', '--name-only', 'HEAD', '--']), 'tracked changes') + require(not gitrun(['diff', '--no-ext-diff', '--no-textconv', '--cached', '--name-only', '--']), 'tracked index changes') # The explicit inventory seal is the disposal evidence for untracked and ignored files. gitrun(['status', '--porcelain=v1', '--untracked-files=all', '--ignored']) remote = gitrun(['ls-remote', '--refs', 'origin']) @@ -338,40 +349,23 @@ def records(self): yield name, read_json(self.meta, name) -def inspect(root, record, roots, age, processes=process_check): - root.recheck() - path = record.get('path') - parts = relative(path) - require(record.get('version') == 1 and record.get('session') == root.owner['session'] and - record.get('kind') in KINDS, 'ownership/disposal record unavailable') - seal_name = hashlib.sha256(path.encode()).hexdigest() + '.json' - require(read_json(root.meta, seal_name) == record, 'disposal seal changed') - keep_check(root.fd, path) - full = root.path + '/' + path - mount_check(full) - require(not within(os.getcwd(), [full]) and full != os.path.expanduser('~') and - not any(r == full or r.startswith(full + '/') for r in roots), 'protected live checkout or root') - with descend(root.fd, parts[:-1]) as parent: - st = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) - require(identity(st) == record.get('identity'), 'candidate replaced') - require(st.st_uid == os.getuid(), 'candidate ownership changed') - require(stat.S_ISDIR(st.st_mode) or (record['kind'] == 'transcript' and stat.S_ISREG(st.st_mode)), 'candidate must be a real directory or transcript file') - if stat.S_ISDIR(st.st_mode): - with descend(parent, parts[-1:]) as child: - contents = inventory(child) - else: - contents = {'': file_snapshot(parent, parts[-1])} - require(contents == record.get('inventory'), 'content changed or lacks explicit disposal evidence') - newest = max([record.get('sealed', time.time()), st.st_mtime] + [v[4] / 1e9 for v in contents.values()]) - require(time.time() - newest >= age, 'younger than age threshold') - # A repository marker is allowed only at the candidate root, and only for Git-aware classes. - is_git = '.git' in contents - git = None - if record['kind'] in ('worktree', 'review') and is_git: - git = git_check(full, roots, record['kind']) - else: - require(not any(PurePosixPath(p).name == '.git' for p in contents), 'nested worktree or repository') - require(record['kind'] != 'worktree', 'worktree evidence missing') +def logical_bytes(contents): + return sum(value[3] for value in contents.values() if stat.S_ISREG(value[2])) + + +def reject_saved_git(contents, verified_git): + children = {} + for path in contents: + parent, _, name = path.rpartition('/') + children.setdefault(parent, set()).add(name) + for parent, names in children.items(): + signature = (('HEAD' in names and bool(names & {'objects', 'index', 'commondir', 'gitdir'})) or + ('objects' in names and bool(names & {'refs', 'packed-refs'}))) + require(not signature or (verified_git and parent == '.git'), + 'unsupported bare or saved Git directory: ' + (parent or '.')) + + +def worktree_eligibility(record, roots): if record['kind'] in ('scratch', 'cache'): base = record.get('base') require(isinstance(base, str) and within(base, roots), 'base evidence outside authorized roots or missing') @@ -408,6 +402,46 @@ def inspect(root, record, roots, age, processes=process_check): except FileNotFoundError: absent = True require(absent and removed not in users, 'associated worktree is not removed') + + +def inspect(root, record, roots, age, processes=process_check, min_transcript_bytes=MIN_TRANSCRIPT_BYTES): + root.recheck() + path = record.get('path') + parts = relative(path) + require(record.get('version') == 1 and record.get('session') == root.owner['session'] and + record.get('kind') in KINDS, 'ownership/disposal record unavailable') + seal_name = hashlib.sha256(path.encode()).hexdigest() + '.json' + require(read_json(root.meta, seal_name) == record, 'disposal seal changed') + keep_check(root.fd, path) + full = root.path + '/' + path + mount_check(full) + require(not within(os.getcwd(), [full]) and full != os.path.expanduser('~') and + not any(r == full or r.startswith(full + '/') for r in roots), 'protected live checkout or root') + with descend(root.fd, parts[:-1]) as parent: + st = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) + require(identity(st) == record.get('identity'), 'candidate replaced') + require(st.st_uid == os.getuid(), 'candidate ownership changed') + require(stat.S_ISDIR(st.st_mode) or (record['kind'] == 'transcript' and stat.S_ISREG(st.st_mode)), 'candidate must be a real directory or transcript file') + if stat.S_ISDIR(st.st_mode): + with descend(parent, parts[-1:]) as child: + contents = inventory(child) + else: + contents = {'': file_snapshot(parent, parts[-1])} + require(contents == record.get('inventory'), 'content changed or lacks explicit disposal evidence') + newest = max([record.get('sealed', time.time()), st.st_mtime] + [v[4] / 1e9 for v in contents.values()]) + require(time.time() - newest >= age, 'younger than age threshold') + # A repository marker is allowed only at the candidate root, and only for Git-aware classes. + is_git = '.git' in contents + git = None + if record['kind'] in ('worktree', 'review') and is_git: + git = git_check(full, roots, record['kind']) + else: + require(not any(PurePosixPath(p).name == '.git' for p in contents), 'nested worktree or repository') + require(record['kind'] != 'worktree', 'worktree evidence missing') + reject_saved_git(contents, git is not None) + require(record['kind'] != 'transcript' or logical_bytes(contents) >= min_transcript_bytes, + 'transcript below minimum logical bytes') + worktree_eligibility(record, roots) processes([full] + ([git['gitdir']] if git else [])) return contents, git @@ -433,8 +467,8 @@ def erase(fd, expected, prefix='', before_unlink=None): os.unlink(name, dir_fd=fd) -def remove(root, record, roots, age, processes=process_check): - contents, git = inspect(root, record, roots, age, processes) +def remove(root, record, roots, age, processes=process_check, min_transcript_bytes=MIN_TRANSCRIPT_BYTES): + contents, git = inspect(root, record, roots, age, processes, min_transcript_bytes=min_transcript_bytes) parts = relative(record['path']) token = '.jpdev-clean-removing-' + uuid.uuid4().hex with descend(root.fd, parts[:-1]) as parent: @@ -471,6 +505,7 @@ def remove(root, record, roots, age, processes=process_check): root.recheck() def check(): root.recheck() + worktree_eligibility(record, roots) mount_check(moved) if git and not within(after['gitdir'], [moved]): require(git_metadata(after['gitdir']) == after['metadata'] and @@ -484,6 +519,7 @@ def check(): processes([moved, root.path + '/' + record['path']] + ([git['gitdir']] if git else [])) check() erase(child, contents, before_unlink=check) + check() require(identity(os.stat(token, dir_fd=parent, follow_symlinks=False)) == record['identity'], 'quarantine replaced') os.rmdir(token, dir_fd=parent) except BaseException: @@ -500,6 +536,12 @@ def duration(text): return int(match[1]) * dict(s=1, m=60, h=3600, d=86400, w=604800)[match[2]] +def byte_count(text): + if not re.fullmatch(r'[0-9]+', text): + raise argparse.ArgumentTypeError('byte count must be a nonnegative integer') + return int(text) + + def initialize(path, session): require(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_-]{7,79}', session), 'session token must be 8..80 letters, digits, underscores or hyphens') if not os.path.exists(path): @@ -543,6 +585,8 @@ def parser(): result.add_argument('action', nargs='?', choices=('init', 'seal')) result.add_argument('--root', action='append', default=[], help='explicit root; repeat to authorize Git evidence roots') result.add_argument('--older-than', type=duration, default=7 * 86400, metavar='DURATION') + result.add_argument('--min-transcript-bytes', type=byte_count, default=MIN_TRANSCRIPT_BYTES, metavar='N', + help='transcript-only minimum logical regular-file bytes, inclusive (default: 10485760; 0 disables)') result.add_argument('--yes', action='store_true', help='remove verified candidates (default: dry run)') result.add_argument('--session') result.add_argument('--path') @@ -585,14 +629,14 @@ def main(argv=None): records = list(root.records()) for name, record in records: try: - contents, _ = inspect(root, record, roots, args.older_than) - bytes_used = sum(v[3] for v in contents.values() if stat.S_ISREG(v[2])) + contents, _ = inspect(root, record, roots, args.older_than, min_transcript_bytes=args.min_transcript_bytes) + bytes_used = logical_bytes(contents) total += 1 size += bytes_used print(json.dumps({'root': path, 'path': record['path'], 'bytes': bytes_used, 'reason': 'sealed ' + record['kind'], 'outcome': 'candidate'}), flush=True) if args.yes: - remove(root, record, roots, args.older_than) + remove(root, record, roots, args.older_than, min_transcript_bytes=args.min_transcript_bytes) os.unlink(name, dir_fd=root.meta) removed += 1 print(json.dumps({'root': path, 'path': record['path'], 'outcome': 'removed'}), flush=True) diff --git a/tests/clean_test.py b/tests/clean_test.py index 3b620f1..4644636 100644 --- a/tests/clean_test.py +++ b/tests/clean_test.py @@ -5,6 +5,7 @@ import io import hashlib import stat +import shutil import json import os from pathlib import Path @@ -60,7 +61,7 @@ def candidate(self, kind='review', path='old', base=None, removed=None): return folder, record def remove(self, record, processes=lambda paths: None): - clean.remove(self.owner, record, self.roots, 0, processes) + clean.remove(self.owner, record, self.roots, 0, processes, min_transcript_bytes=0) def refused(self, record, reason, processes=lambda paths: None): with self.assertRaisesRegex((clean.Refusal, OSError), reason): @@ -149,7 +150,7 @@ def test_dry_run_and_absent_root(self): folder, record = self.candidate() with patch.object(clean, 'process_check', lambda paths: None), contextlib.redirect_stdout(io.StringIO()) as output: # Defaults bind at definition, so replace only the live evidence seam here. - with patch.object(clean, 'inspect', lambda *a: (record['inventory'], None)): + with patch.object(clean, 'inspect', lambda *a, **kw: (record['inventory'], None)): self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s']), 0) self.assertTrue(folder.exists()) self.assertIn('"bytes":', output.getvalue()) @@ -368,10 +369,10 @@ def test_transcript_file(self): def test_main_removal_and_error_status(self): folder, record = self.candidate() inspect, remove = clean.inspect, clean.remove - def inspect_fixture(*args): - return inspect(*args[:4], processes=lambda paths: None) - def remove_fixture(*args): - return remove(*args, processes=lambda paths: None) + def inspect_fixture(*args, **kwargs): + return inspect(*args[:4], processes=lambda paths: None, **kwargs) + def remove_fixture(*args, **kwargs): + return remove(*args, processes=lambda paths: None, **kwargs) with patch.object(clean, 'inspect', inspect_fixture), patch.object(clean, 'remove', remove_fixture): with contextlib.redirect_stdout(io.StringIO()) as output: self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', '--yes']), 0) @@ -563,8 +564,8 @@ def test_lock_and_absent_seal(self): folder, record = self.candidate() inspect, remove = clean.inspect, clean.remove clean.fcntl.flock(self.owner.meta, clean.fcntl.LOCK_EX | clean.fcntl.LOCK_NB) - with patch.object(clean, 'inspect', lambda *args: inspect(*args[:4], processes=lambda paths: None)), \ - patch.object(clean, 'remove', lambda *args: remove(*args, processes=lambda paths: None)): + with patch.object(clean, 'inspect', lambda *args, **kw: inspect(*args[:4], processes=lambda paths: None, **kw)), \ + patch.object(clean, 'remove', lambda *args, **kw: remove(*args, processes=lambda paths: None, **kw)): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', '--yes']), 1) self.assertTrue(folder.exists()) @@ -683,6 +684,193 @@ def test_unlink_failure(self): with patch.object(clean.os, 'unlink', side_effect=PermissionError('removal denied')): self.refused(record, 'removal denied') + def test_bare_git_refused(self): + for nested in (False, True): + with self.subTest(nested=nested): + path = 'nested-review' if nested else 'private.git' + repo = self.root / path / 'saved.git' if nested else self.root / path + repo.parent.mkdir(parents=True, exist_ok=True) + self.git('init', '--bare', str(repo)) + tree = self.git('--git-dir=' + str(repo), 'mktree') + commit = self.git('--git-dir=' + str(repo), 'commit-tree', tree, '-m', 'unpublished') + self.git('--git-dir=' + str(repo), 'update-ref', 'refs/heads/private', commit) + _, record = self.candidate(path=path) + self.refused(record, 'bare or saved Git directory') + self.assertEqual(self.git('--git-dir=' + str(repo), 'rev-parse', 'refs/heads/private'), commit) + + def test_saved_git_refused(self): + folder, _, _ = self.repository(review=True) + (folder / 'file').write_text('private commit') + self.git('commit', '-am', 'unpublished', cwd=folder) + for nested in (False, True): + with self.subTest(nested=nested): + path = 'saved-nested' if nested else 'saved-root' + target = self.root / path + if nested: + target.mkdir() + target = target / 'metadata' + # Copy only disposable fixture metadata; no repository is adopted. + shutil.copytree(folder / '.git', target) + _, record = self.candidate(path=path) + self.refused(record, 'bare or saved Git directory') + self.assertTrue((target / 'objects').exists()) + + def test_inherited_alternate_index_refused(self): + folder, _, _ = self.repository(review=True) + alternate = self.home / 'published.index' + alternate.write_bytes((folder / '.git/index').read_bytes()) + (folder / 'file').write_text('private staged contents') + self.git('add', 'file', cwd=folder) + (folder / 'file').write_text('published') + self.assertEqual(self.git('diff', '--cached', '--name-only', cwd=folder), 'file') + _, record = self.candidate() + with patch.dict(os.environ, {'GIT_INDEX_FILE': str(alternate)}): + self.refused(record, 'inherited Git evidence settings') + self.assertEqual(self.git('diff', '--cached', '--name-only', cwd=folder), 'file') + + def test_inherited_git_indirection_refused(self): + self.repository(review=True) + _, record = self.candidate() + for name in ('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_OBJECT_DIRECTORY', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG', 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_PARAMETERS', 'GIT_NAMESPACE', 'GIT_SHALLOW_FILE', 'GIT_EXEC_PATH', + 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_SYSTEM', 'GIT_UNKNOWN_SETTING'): + with self.subTest(name=name), patch.dict(os.environ, {name: 'fixture-value'}): + self.refused(record, 'inherited Git evidence settings: ' + name) + + def hidden_index_flag(self, flag): + folder, _, _ = self.repository(review=True) + self.git('update-index', '--' + flag, 'file', cwd=folder) + (folder / 'file').write_text('private hidden tracked contents') + self.assertEqual(self.git('diff', '--name-only', 'HEAD', cwd=folder), '') + _, record = self.candidate() + self.refused(record, 'unsupported Git index flags') + self.assertEqual((folder / 'file').read_text(), 'private hidden tracked contents') + + def test_assume_unchanged_refused(self): + self.hidden_index_flag('assume-unchanged') + + def test_skip_worktree_refused(self): + self.hidden_index_flag('skip-worktree') + + def eligibility_race(self, kind, during_erase): + old, base, _ = self.repository(branch=False) + self.git('worktree', 'remove', str(old), cwd=base) + path = 'base/.jpdev-shared' if kind == 'cache' else 'builder-scratch' + folder, record = self.candidate(kind, path=path, base=str(base), removed=str(old)) + recreated = self.root / 'new-worktree' if kind == 'cache' else old + if during_erase: + (folder / 'a-first').write_text('disposable first entry') + _, record = self.candidate(kind, path=path, base=str(base), removed=str(old)) + original = clean.os.unlink if during_erase else clean.os.rename + fired = False + def create(): + nonlocal fired + if not fired: + fired = True + self.git('worktree', 'add', '--detach', str(recreated), cwd=base) + def rename(src, dst, *args, **kwargs): + result = original(src, dst, *args, **kwargs) + if os.fspath(dst).startswith('.jpdev-clean-removing-'): + create() + return result + def unlink(name, *args, **kwargs): + result = original(name, *args, **kwargs) + if name == 'a-first': + create() + return result + with patch.object(clean.os, 'unlink', unlink) if during_erase else patch.object(clean.os, 'rename', rename): + self.refused(record, 'worktree users|not removed') + self.assertTrue(fired) + self.assertTrue(recreated.exists()) + self.assertEqual((folder / 'report').read_text(), 'session-created evidence') + + def test_cache_quarantine_race(self): + self.eligibility_race('cache', False) + + def test_cache_erase_race(self): + self.eligibility_race('cache', True) + + def test_scratch_quarantine_race(self): + self.eligibility_race('scratch', False) + + def test_scratch_erase_race(self): + self.eligibility_race('scratch', True) + + def test_ambient_global_git_config_isolated(self): + folder, _, _ = self.repository(review=True) + _, record = self.candidate() + (self.home / '.gitconfig').write_text('invalid config syntax\n') + with patch.dict(os.environ, {'HOME': str(self.home)}): + self.remove(record) + self.assertFalse(folder.exists()) + + def test_saved_partial_git_metadata_refused(self): + folder, _ = self.candidate(path='saved-admin') + (folder / 'HEAD').write_text('') + (folder / 'index').write_text('') + _, record = self.candidate(path='saved-admin') + self.refused(record, 'bare or saved Git directory') + + def test_transcript_threshold_cli(self): + folder, record = self.candidate('transcript', path='logs') + inspect, remove = clean.inspect, clean.remove + def inspect_fixture(*args, **kwargs): + return inspect(*args[:4], processes=lambda paths: None, **kwargs) + def remove_fixture(*args, **kwargs): + return remove(*args, processes=lambda paths: None, **kwargs) + with patch.object(clean, 'inspect', inspect_fixture), patch.object(clean, 'remove', remove_fixture): + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s']), 1) + self.assertIn('transcript below minimum', output.getvalue()) + self.assertTrue(folder.exists()) + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(clean.main(['--root', str(self.root), '--older-than', '0s', + '--min-transcript-bytes', '0', '--yes']), 0) + self.assertIn('"outcome": "removed"', output.getvalue()) + self.assertFalse(folder.exists()) + + def test_transcript_threshold(self): + self.assertEqual(clean.parser().parse_args([]).min_transcript_bytes, 10485760) + for value in ('-1', '1.5', '10MiB', '', 'nan'): + with self.subTest(invalid=value), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + clean.parser().parse_args(['--min-transcript-bytes', value]) + self.assertEqual(clean.parser().parse_args(['--min-transcript-bytes', '0']).min_transcript_bytes, 0) + for size, floor in ((0, 0), (0, 4), (3, 4), (4, 4), (5, 4), (4, 10485760)): + with self.subTest(size=size, floor=floor): + path = 'log-%s-%s' % (size, floor) + log = self.root / path + log.write_bytes(b'x' * size) + args = clean.parser().parse_args(['seal', '--root', str(self.root), '--session', 'session-test', + '--path', path, '--kind', 'transcript']) + clean.seal(self.owner, args) + record = next(r for _, r in self.owner.records() if r['path'] == path) + if size < floor: + with self.assertRaisesRegex(clean.Refusal, 'transcript below minimum'): + clean.remove(self.owner, record, self.roots, 0, lambda paths: None, floor) + self.assertTrue(log.exists()) + else: + clean.remove(self.owner, record, self.roots, 0, lambda paths: None, floor) + self.assertFalse(log.exists()) + + def test_transcript_directory_threshold_and_other_kinds(self): + folder, _ = self.candidate('transcript', path='logs') + (folder / 'report').write_bytes(b'xx') + (folder / 'nested').mkdir() + (folder / 'nested' / 'log').write_bytes(b'xx') + args = clean.parser().parse_args(['seal', '--root', str(self.root), '--session', 'session-test', + '--path', 'logs', '--kind', 'transcript']) + clean.seal(self.owner, args) + record = next(r for _, r in self.owner.records() if r['path'] == 'logs') + with self.assertRaisesRegex(clean.Refusal, 'transcript below minimum'): + clean.remove(self.owner, record, self.roots, 0, lambda paths: None, 5) + self.assertTrue(folder.exists()) + clean.remove(self.owner, record, self.roots, 0, lambda paths: None, 4) + self.assertFalse(folder.exists()) + folder, record = self.candidate('review', path='small-review') + clean.remove(self.owner, record, self.roots, 0, lambda paths: None, 10485760) + self.assertFalse(folder.exists()) + if __name__ == '__main__': unittest.main()