From 56ad570019b997f3495c0360df8465a0762c948a Mon Sep 17 00:00:00 2001 From: Nishad <133812901+nishu-builder@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:37:53 +0000 Subject: [PATCH 1/2] Push source stack branches through the CAOS server --- SPEC.md | 4 + design/agent-github.md | 91 ++----- design/agent-publish.md | 320 ++++++----------------- design/agent-stacks.md | 27 +- std/github/src/main.rs | 2 +- std/llm-step/src/main.rs | 27 +- std/llm-step/src/publish_source.rs | 129 ++++++---- std/llm-step/src/push_stack.rs | 398 +++++++++++++++++++++++++++++ std/llm-step/src/source_trees.rs | 6 +- std/llm-step/src/stack.rs | 36 ++- std/llm-step/src/tools.rs | 1 + 11 files changed, 675 insertions(+), 366 deletions(-) create mode 100644 std/llm-step/src/push_stack.rs diff --git a/SPEC.md b/SPEC.md index b345a90a7..88e0bcf36 100644 --- a/SPEC.md +++ b/SPEC.md @@ -642,3 +642,7 @@ See [agent publication](design/agent-publish.md) for mechanics and recovery. The stack tool registers ordered source gitlinks and their original predecessor boundaries. Rebase replays linear histories with object-level Git merges; merge mode propagates updated predecessors without rewriting history. A conflict keeps original pointers unchanged and records a separate editable draft plus the full conflict report. Continue explicitly acknowledges resolution. Completed operations replace every source pointer together after checking the original inputs. No .caos/conflicts is introduced by this workflow. See [Agent stacks](design/agent-stacks.md). + +The push_stack tool pins all source commits and remote-head leases, then publishes each branch through the same server path as publish_source. A rebased push explicitly enables rewrite; publication +retains the exact lease and all ordinary guards. Each branch outcome is recorded before proceeding. Recovery skips completed branches and uses the original pins. Failure stops later pushes and returns +partial receipts. PR creation and GitHub stack membership are separate from pushing. diff --git a/design/agent-github.md b/design/agent-github.md index 163ab6dc4..aeaf42c50 100644 --- a/design/agent-github.md +++ b/design/agent-github.md @@ -1,14 +1,12 @@ # Importing and publishing -Imports are implemented through the server endpoint, `caos import-git`, and -the agent's `import_source` tool. Publication proceeds through branches, PRs -and stacks. Separate merge drafts remain a follow-up. +Imports are implemented through the server endpoint, `caos import-git`, and the agent's `import_source` tool. Publication pushes individual branches or registered stacks of branches. Stack operations +keep merge drafts outside source history. ## Importing -`import_source(source, revision?, into)` runs inline in `std/llm-step`. -It accepts an HTTPS repository and a branch, full ref, or full commit hash. -Omitting `revision` selects the default branch. Keep `/import` for local paths. +`import_source(source, revision?, into)` runs inline in `std/llm-step`. It accepts an HTTPS repository and a branch, full ref, or full commit hash. Omitting `revision` selects the default branch. Keep +`/import` for local paths. For each tool call: @@ -27,18 +25,12 @@ For each tool call: imports/repo/main.source.json provenance ``` -The destination and provenance path must both be unused. An import creates -an unchanged snapshot; it does not merge into or advance another source. -Provenance records the repository, requested revision, commit, observation -time, and default branch when known. For `origin/main`, choose the repository -from the selected source's provenance and import `main` at a fresh path. +The destination and provenance path must both be unused. An import creates an unchanged snapshot; it does not merge into or advance another source. Provenance records the repository, requested +revision, commit, observation time, and default branch when known. For `origin/main`, choose the repository from the selected source's provenance and import `main` at a fresh path. -The [server endpoint](git-import.md) fetches H and its full history into private -staging, verifies them, and publishes the complete pack into the server store. It uses verified complete imports as -negotiation tips; standalone trees and blobs may still be downloaded again. -A completion marker for the same URL and H skips fetch and verification. -The endpoint handles object availability; callers handle ref resolution and -conversation state. +The [server endpoint](git-import.md) fetches H and its full history into private staging, verifies them, and publishes the complete pack into the server store. It uses verified complete imports as +negotiation tips; standalone trees and blobs may still be downloaded again. A completion marker for the same URL and H skips fetch and verification. The endpoint handles object availability; callers +handle ref resolution and conversation state. Supply the token through the existing secret store: @@ -50,64 +42,21 @@ reader=std/llm-step reader=std/github ``` -Keep the value file ignored and run `caos secrets` to initialize its entropy. -The agent uses `/secret/github-token` for GitHub ref lookup and passes -`--github-token-file=/secret/github-token` to `import-git`. The command -forwards it in the sensitive `X-Caos-Git-Token` header; the server does not look -up the calling job's secrets. +Keep the value file ignored and run `caos secrets` to initialize its entropy. The agent uses `/secret/github-token` for GitHub ref lookup and passes `--github-token-file=/secret/github-token` to +`import-git`. The command forwards it in the sensitive `X-Caos-Git-Token` header; the server does not look up the calling job's secrets. -Ref lookup and fetch share a repository-scoped Git credential helper. Tokens -stay out of URLs, Git config, saved arguments, provenance, and logs. Automatic -GitHub credentials apply only to `github.com` on the default HTTPS port. -Public imports need no token. Importing needs neither `gh` nor another worker. +Ref lookup and fetch share a repository-scoped Git credential helper. Tokens stay out of URLs, Git config, saved arguments, provenance, and logs. Automatic GitHub credentials apply only to +`github.com` on the default HTTPS port. Public imports need no token. Importing needs neither `gh` nor another worker. ## Publishing -The [publication design](agent-publish.md) separates the remaining work: +[Branch publication](agent-publish.md) uses POST /git/push, caos push-git and publish_source to push exact commits directly from the server with an expected-head lease. The github tool runs gh with an +explicit repository for PRs, issues, review and stack metadata. -- **Branches:** `POST /git/push`, `caos push-git` and `publish_source` push - exact code commits from the server with a lease. Each layer has its own - implementation PR. -- **PRs:** separate PRs add the `std/github` worker and agent `github` tool. Next, validate the agent's create, update and review workflow using - `gh`, with explicit repositories and branch names. -- **Stacks:** keep code boundaries in gitlinks, publish bottom to top, and - use `gh stack link` with existing PR URLs. Validate linking, propagation - and landing without a source checkout in the GitHub worker. +[Stack operations](agent-stacks.md) register ordered source gitlinks, remember each layer's predecessor, and merge or rebase them using Git objects. Conflicts pause with a separate draft gitlink and +report. The agent edits the draft and explicitly continues; finished source history contains no .caos/conflicts or draft editing commits. -The TUI's `/pr` and `/publish-branch` commands are removed. The follow-ups -start with workflow instructions and integration tests using these tools. +push_stack pushes registered layers through the same server path as publish_source, recording each branch's result. Stack pushing needs no GitHub worker and creates no PRs or GitHub stack membership. +PR automation is a separate follow-up. -### Merges - -This section is a follow-up proposal. Current merges still use source-tree -conflict ledgers and the publication guards described in SPEC.md. - -Clean merges use the existing merge worker. On conflict, preserve the source O -and keep the attempt beside it in the conversation: - -```text -feature/01-core gitlink -> O -merges/update-main/ - ours gitlink -> O - theirs gitlink -> T - work gitlink -> D - conflicts Git's complete conflict report -``` - -D starts with Git's proposed merged tree and O as its single parent. The agent -edits this separate draft gitlink and tests it. The report stays outside the -code tree; newly created sources and drafts contain no `.caos/conflicts`. - -`finish_merge(attempt="merges/update-main", target="feature/01-core")` takes -the draft's current tree R and creates `M = commit(tree=R, parents=[O,T])`. -Verify O and T against the attempt's creation record, recheck the draft, and -advance the target only if it still points to O. Otherwise retain the result -for reconciliation. Draft commits stay in conversation history, outside M's -ancestry. Test the final source before publication. - -Finishing explicitly asserts resolution. Preserve Git's complete conflict -report, including messages and stage objects; deleting markers or report rows -is not proof that structural conflicts are resolved. Delegate by copying the -whole attempt with `cp -a`, harvesting the edited draft, and then finishing. -Abandoning an attempt leaves the source unchanged. Handle old source-tree -conflict ledgers before removing their compatibility cleanup. +The TUI's /pr and /publish-branch commands are removed. /import remains for local paths. Older non-stack merge operations still use the legacy source-tree conflict ledger. diff --git a/design/agent-publish.md b/design/agent-publish.md index 3971cc1dc..85c073bcb 100644 --- a/design/agent-publish.md +++ b/design/agent-publish.md @@ -1,171 +1,108 @@ -# Agent publication and PR stacks +# Agent publication -Importing is implemented. Publication builds on it in three steps: - -| Step | What ships | Remaining work | -| --- | --- | --- | -| Branches | Server push endpoint, `caos push-git`, and `publish_source` in separate implementation PRs. | Merge the branch-publication implementation. | -| PRs | The stack adds `std/github` and the agent's `github` tool in separate PRs. | Validate creating, updating and reviewing a PR through the agent. | -| Stacks | The worker includes `gh-stack`. Source gitlinks already carry stack boundaries. | Validate publication, linking, updates and landing of a dependent stack. | - -The GitHub worker is tested for execution, secret grants and retry handling. -Live PR creation and stack linking remain to be exercised. Those follow-ups -should start with agent instructions and integration tests; the operations -below use the existing tools. +CAOS publishes exact code commits from its server. The GitHub worker manages PR metadata. See [agent-stacks.md](agent-stacks.md) for stack registration, restacking, conflict resolution and branch +publication. ## Branches -The server publishes code commits directly from its bare Git store, without -checking out source files. +The server publishes code commits directly from its bare Git store, without checking out source files. -| Layer | Interface | -| --- | --- | -| Server | `POST /git/push {destination, commit, branch, expected}` | -| Worker command | `caos push-git --expected=` | -| Agent tool | `publish_source(source_tree, repository, branch)` | +| Layer | Interface | | --- | --- | | Server | `POST /git/push {destination, commit, branch, expected, rewrite?}` | | Worker command | `caos push-git +--expected=` | | Agent tool | `publish_source(source_tree, repository, branch, rewrite?)` | The request fields are: -| Field | Meaning | -| --- | --- | -| `destination` | Remote repository's HTTPS URL, e.g. `https://github.com/owner/repo.git`. | -| `commit` | Full hash H of the code commit to publish, already stored in CAOS. | -| `branch` | Destination branch in that remote, e.g. `feature/parser`, without `refs/heads/`. | -| `expected` | Full hash E expected at that same remote branch, or JSON `null` if it must not exist. Required; the CLI spells `null` as `absent`. | +| Field | Meaning | | --- | --- | | `destination` | Remote repository's HTTPS URL, e.g. `https://github.com/owner/repo.git`. | | `commit` | Full hash H of the code commit to publish, already stored in +CAOS. | | `branch` | Destination branch in that remote, e.g. `feature/parser`, without `refs/heads/`. | | `expected` | Full hash E expected at that same remote branch, or JSON `null` if it must not +exist. Required; the CLI spells `null` as `absent`. | | `rewrite` | Optional boolean, default false. Permit a non-fast-forward update while still requiring the exact expected remote head. | -`refs/heads/feature/parser` is Git's full name for the branch `feature/parser`. -Ordinary branch pushes can infer this prefix from a local branch. Since CAOS -pushes a commit hash, it explicitly names the remote branch: +`refs/heads/feature/parser` is Git's full name for the branch `feature/parser`. Ordinary branch pushes can infer this prefix from a local branch. Since CAOS pushes a commit hash, it explicitly names +the remote branch: ```text git push H:refs/heads/feature/parser ``` -This is [standard Git refspec syntax](https://git-scm.com/docs/git-push). -No corresponding branch is needed or created in CAOS. This endpoint publishes -branches only; it does not publish tags or delete refs. +This is [standard Git refspec syntax](https://git-scm.com/docs/git-push). No corresponding branch is needed or created in CAOS. This endpoint publishes branches only; it does not publish tags or +delete refs. The endpoint performs one push: 1. Validate the HTTPS destination, branch and full hashes; require H to be a stored commit. Trust the complete history verified at ingestion and startup. 2. If E is non-null, require E to be a stored ancestor of H. If H contains E, - CAOS already has E. A missing E or non-fast-forward is a rejection. + CAOS already has E. A missing E is a rejection. An explicit rewrite: true skips + the ancestry check for an intentional rebased-history update. Reject H if its tree contains paths matched by its own .gitignore rules. 3. Push H to the destination branch with --force-with-lease=refs/heads/:, disabling tag following. - Empty E requires creation. The ancestry check prevents history rewrites; - despite the Git flag's name, this API permits only creates and fast-forwards. - There is no force-push option. Duplicate requests to the same branch are serialized. + Empty E requires creation. The default permits only creates and fast-forwards; + rewrite: true allows a history rewrite while retaining that exact lease. + Duplicate requests to the same branch are serialized. 4. Return complete, conflict, or uncertain, with a reason. Known validation and per-ref receiver rejections are definite failures. Unconfirmed transport failures are uncertain. The CLI preserves these results for llm-step. -The endpoint does not fetch, import, merge, rebase, rewrite commits, or perform -a follow-up remote lookup. Objects transfer directly from CAOS to the -destination through Git. Reuse import authentication: token-file option, -sensitive header and repository-scoped credential helper. - -llm-step owns the workflow. It selects a source gitlink, resolves conflicts, -tests and inspects the diff, then reads its commit H and the remote head E. -Before sending, it records H, E, destination and branch under the tool-call -identity. Every attempt may send that same pinned intent; recovery never -substitutes a newer commit or refreshes the lease. Publishing leaves the -source gitlink unchanged. - -On a lease conflict, llm-step can separately import the remote head, merge or -rebase, test, and make a new publication call. Other rejections carry their -reason through the CLI to the tool result. Importing and integration are -never hidden inside a push. - -A remote can accept a push before the connection drops. Git's HTTP retry can -then report a stale lease. After a receiver conflict or uncertain result, -llm-step reads the branch: H confirms completion. Otherwise it preserves a definite rejection; -for uncertainty, another value than E is a conflict, while E or a failed lookup -remains uncertain because a push may still be running. The endpoint itself does no recovery. A success receipt records the -original push even if the branch later advances. - -Publication transfers the exact commit. Before pushing, the server checks H's -tree against its versioned .gitignore files, including nested rules and -negations. A match returns HTTP 422 with code `ignored-files`; the CLI and -agent retain this as a definite rejection without remote reconciliation. Git -performs the check using a private index; no source files are checked out. - -This is deliberately stricter than ordinary Git: a tracked file matching an -ignore rule is rejected too. Global excludes and .git/info/exclude do not -apply. This checks the requested snapshot only, not earlier commits; a file -added and deleted in its history is outside this check. The server never -strips files or rewrites history. - -Local Git staging still respects .gitignore for untracked files. Imports keep -their exact commits, and agent tools continue to capture files as they do -today. Ignored scratch files can therefore remain in a source during work; -the agent must remove them or adjust the rules before publication. - -Merge conflicts currently create a tracked .caos/conflicts ledger inside a -source tree, including conflicts without inline markers. llm-step must resolve -and clear it before publishing. The generic endpoint does not scan for that -ledger, inline markers, or conversation ancestry. +The endpoint does not fetch, import, merge, rebase, rewrite commits, or perform a follow-up remote lookup. Objects transfer directly from CAOS to the destination through Git. Reuse import +authentication: token-file option, sensitive header and repository-scoped credential helper. + +llm-step owns the workflow. It selects a source gitlink, resolves conflicts, tests and inspects the diff, then reads its commit H and the remote head E. Before sending, it records H, E, destination +and branch under the tool-call identity. Every attempt may send that same pinned intent; recovery never substitutes a newer commit or refreshes the lease. Publishing leaves the source gitlink +unchanged. + +On a lease conflict, llm-step can separately import the remote head, merge or rebase, test, and make a new publication call. Other rejections carry their reason through the CLI to the tool result. +Importing and integration are never hidden inside a push. + +A remote can accept a push before the connection drops. Git's HTTP retry can then report a stale lease. After a receiver conflict or uncertain result, llm-step reads the branch: H confirms completion. +Otherwise it preserves a definite rejection; for uncertainty, another value than E is a conflict, while E or a failed lookup remains uncertain because a push may still be running. The endpoint itself +does no recovery. A success receipt records the original push even if the branch later advances. + +Publication transfers the exact commit. Before pushing, the server checks H's tree against its versioned .gitignore files, including nested rules and negations. A match returns HTTP 422 with code +`ignored-files`; the CLI and agent retain this as a definite rejection without remote reconciliation. Git performs the check using a private index; no source files are checked out. + +This is deliberately stricter than ordinary Git: a tracked file matching an ignore rule is rejected too. Global excludes and .git/info/exclude do not apply. This checks the requested snapshot only, +not earlier commits; a file added and deleted in its history is outside this check. The server never strips files or rewrites history. + +Local Git staging still respects .gitignore for untracked files. Imports keep their exact commits, and agent tools continue to capture files as they do today. Ignored scratch files can therefore +remain in a source during work; the agent must remove them or adjust the rules before publication. + +Registered stacks keep conflicts in a separate draft gitlink and report, outside source history. Older merge operations can still create a .caos/conflicts ledger; resolve it before publishing. The +generic endpoint does not scan for that ledger, inline markers, or conversation ancestry. ## PRs -`std/github` contains Git, `gh`, and the pinned `github/gh-stack` v0.1.1 -extension. It is registered as a built-in tool, available without a project-defined `caos-tools` entry. +`std/github` contains Git, `gh`, and the pinned `github/gh-stack` v0.1.1 extension. It is registered as a built-in tool, available without a project-defined `caos-tools` entry. -The tool is `github(repository, args, stdin?)`. `args` is an argument array passed -directly to `gh`; it is never interpolated into a shell command. Return exit -status, stdout and stderr through ordinary tool results. This covers issues, -comments, PR creation/editing and stack operations without separate wrappers -for each GitHub action. Initially use stdin for bodies (`--body-file -`); +The tool is `github(repository, args, stdin?)`. `args` is an argument array passed directly to `gh`; it is never interpolated into a shell command. Return exit status, stdout and stderr through +ordinary tool results. This covers issues, comments, PR creation/editing and stack operations without separate wrappers for each GitHub action. Initially use stdin for bodies (`--body-file -`); commands needing local file attachments can be added when needed. -The worker sets `GH_REPO` explicitly and reads `GH_TOKEN` from its granted -`/secret/github-token`. Add `reader=std/github` to the existing secret; -`std/llm-step` also needs the grant: it uses the token for import/push and -includes that identity in the model turn’s cache key. The agent carries the -GitHub worker source and evaluates it when called, so secret marking of the -child worker does not alter the agent’s own reader identity. Use an isolated temporary -GitHub configuration and disable prompts. Install the extension in the image. -The worker needs no source checkout or local branches for the operations below. +The worker sets `GH_REPO` explicitly and reads `GH_TOKEN` from its granted `/secret/github-token`. Add `reader=std/github` to the existing secret; `std/llm-step` also needs the grant: it uses the +token for import/push and includes that identity in the model turn’s cache key. The agent carries the GitHub worker source and evaluates it when called, so secret marking of the child worker does not +alter the agent’s own reader identity. Use an isolated temporary GitHub configuration and disable prompts. Install the extension in the image. The worker needs no source checkout or local branches for +the operations below. ### Invocation recovery -Identical GitHub commands can observe different remote state, and comments -must not be posted again when a worker retries. The harness therefore binds -a stable, unique invocation ID into every GitHub request. A new tool call gets -a new ID; resuming the same call retains it. Normal result caching then belongs -to that observation, rather than to the command arguments indefinitely. - -A unique cache key alone does not prevent duplicate execution. Before running -`gh`, the worker atomically claims a small record under -`refs/caos/github/` using the existing Git compare-and-swap -transport. The record binds the entire pinned ArgTree (arguments, worker image, -secret identity and salt) and a unique attempt ID. Recovery dispatches the -stored task, including its original worker image. Reusing an invocation with a -different ArgTree is an error; changed worker or secret identity requires a new -invocation, after reconciling any earlier write. This preserves result-cache -identity and prevents sharing results across credentials. - -A failed claim with no retained ref is a retryable job failure. Once gh has -run, retry output storage and result-CAS bookkeeping up to three times, without -rerunning gh. Exhausted bookkeeping leaves the claim uncertain. - -Only the attempt that owns the claim executes the command. Record the exit -status and output hashes afterwards; completed duplicates return that result. - -If a claim exists without a recorded result, a duplicate reports pending or -uncertain and does not execute `gh`. Never expire or steal that claim based on -elapsed time. A crash between claiming and execution can therefore require -inspection even when nothing happened. This provides at most one wrapper -execution per invocation, not a transaction or exactly-once guarantee at GitHub. - -Apply this rule to all commands, including reads, to avoid classifying arbitrary -`gh api` requests as safe or unsafe. After an uncertain write, the agent uses a -new read invocation to inspect GitHub, then decides the remaining action. -An absent PR or comment is not proof that a still-running command cannot create -it. If reconciliation cannot establish the outcome, leave it uncertain rather -than repeat the write. Failure of a multi-step command can leave partial changes. -The tool's exit status and transcript must not claim that nothing happened. +Identical GitHub commands can observe different remote state, and comments must not be posted again when a worker retries. The harness therefore binds a stable, unique invocation ID into every GitHub +request. A new tool call gets a new ID; resuming the same call retains it. Normal result caching then belongs to that observation, rather than to the command arguments indefinitely. + +A unique cache key alone does not prevent duplicate execution. Before running `gh`, the worker atomically claims a small record under `refs/caos/github/` using the existing Git +compare-and-swap transport. The record binds the entire pinned ArgTree (arguments, worker image, secret identity and salt) and a unique attempt ID. Recovery dispatches the stored task, including its +original worker image. Reusing an invocation with a different ArgTree is an error; changed worker or secret identity requires a new invocation, after reconciling any earlier write. This preserves +result-cache identity and prevents sharing results across credentials. + +A failed claim with no retained ref is a retryable job failure. Once gh has run, retry output storage and result-CAS bookkeeping up to three times, without rerunning gh. Exhausted bookkeeping leaves +the claim uncertain. + +Only the attempt that owns the claim executes the command. Record the exit status and output hashes afterwards; completed duplicates return that result. + +If a claim exists without a recorded result, a duplicate reports pending or uncertain and does not execute `gh`. Never expire or steal that claim based on elapsed time. A crash between claiming and +execution can therefore require inspection even when nothing happened. This provides at most one wrapper execution per invocation, not a transaction or exactly-once guarantee at GitHub. + +Apply this rule to all commands, including reads, to avoid classifying arbitrary `gh api` requests as safe or unsafe. After an uncertain write, the agent uses a new read invocation to inspect GitHub, +then decides the remaining action. An absent PR or comment is not proof that a still-running command cannot create it. If reconciliation cannot establish the outcome, leave it uncertain rather than +repeat the write. Failure of a multi-step command can leave partial changes. The tool's exit status and transcript must not claim that nothing happened. ### PR workflow @@ -182,116 +119,25 @@ For a single PR: 4. Check the PR's URL, head commit and base with `gh pr view --json ...`, and retain the result in the conversation. -Later source edits advance the same branch through `publish_source`, updating -the existing PR. Preserve human-edited titles and descriptions unless an edit -was requested; use `gh pr edit` for requested metadata or base changes. Use -`gh pr view`, `gh pr checks`, and `gh api` to read discussion, review threads -and checks, and the corresponding CLI/API calls for requested replies. -Create ready-for-review PRs by default. A failed PR creation leaves the -successful branch push intact; recovery follows the invocation rules above. - -The PR follow-up should exercise this through the actual agent in a test -repository: publish and open a PR, advance its head, preserve an edited body, -read and answer review feedback, and reconcile an interrupted operation -before proceeding. Keep repeatable failure cases in automated fixtures. -This needs no additional server endpoint or tool for each PR action. +Later source edits advance the same branch through `publish_source`, updating the existing PR. Preserve human-edited titles and descriptions unless an edit was requested; use `gh pr edit` for +requested metadata or base changes. Use `gh pr view`, `gh pr checks`, and `gh api` to read discussion, review threads and checks, and the corresponding CLI/API calls for requested replies. Create +ready-for-review PRs by default. A failed PR creation leaves the successful branch push intact; recovery follows the invocation rules above. ## Stacks -Keep boundaries as ordinary source gitlinks: +A registered stack consists of ordered source gitlinks and a manifest recording each layer's old predecessor. CAOS updates these commits using Git's object-based merge engine and retains paused +conflicts outside source history. See [the stack workflow](agent-stacks.md) for the representation and tool calls. -```text -imports/repo/base -> H -feature/01-core -> A parent H -feature/02-tests -> B parent A -``` +`push_stack(path, repository, rewrite?)` pins all source commits and remote-head leases, then pushes branches bottom to top using the same publication path as `publish_source`. It records each outcome +before proceeding and reports partial progress if a later branch fails. Recovery retains the original pins and skips completed branches. -Create B by copying A with `cp -a` and editing the copy. Git ancestry establishes -that B includes A. Publication receipts map each gitlink's published commit to -a remote branch; GitHub holds PR URLs, bases and stack membership. +Pushing branches is independent of PR creation and GitHub stack membership. The conversation manifest records the intended order; the remote receives branch refs and commit ancestry. Automated PR +submission is a separate follow-up. -Run the PR workflow bottom to top. The first PR targets the chosen mainline -branch; each later PR targets the preceding layer's branch. Before publishing -an upper layer, verify that it contains the exact lower commit just published. -Initial stacks use branches in one GitHub repository. +## Interfaces -Once the PRs exist, link their URLs in order: - -```sh -GH_REPO=owner/repo gh stack link --base main \ - https://github.com/owner/repo/pull/123 \ - https://github.com/owner/repo/pull/124 -``` +The agent uses publication and GitHub tools for PR operations. The TUI's publication commands and preview UI are removed. Existing publication records remain readable. Local imports keep their TUI +command. -With `GH_REPO`, an explicit `--base`, and existing PR URLs, the pinned -[`link` implementation](https://github.com/github/gh-stack/blob/v0.1.1/cmd/link.go) -can use GitHub's API without a checkout or local stack tracking. Branch -arguments take a different path that can push local branches and create PRs. -The worker receives command arguments and PR identifiers; it does not fetch -the source tree. - -`link` can change PR bases. Check the resulting bases and membership even when -it exits with warnings. If GitHub stacks are unavailable, retain the correctly -chained PRs and report that linking was unavailable. - -There is no atomic transaction spanning several pushes, PRs and stack linking. -Record each completed step and reconcile the remainder after a partial failure; -do not roll back successful pushes or delete PRs automatically. - -### Updating and landing - -If A changes to A1, merge A1 into B to make B1, test, then publish A1 and B1 -in that order. Both remote branches advance normally; the PRs and their URLs -stay the same. Apply this propagation upward for additional layers. - -After lower PRs merge, import the actual resulting mainline tip and integrate -it into the next surviving layer. Do not assume squash or rebase merges preserve -A's hash. Inspect the resulting diff so already-landed changes are not proposed -again, retarget the surviving PR if needed, and propagate the update upward. - -Keep clean merges and today's conflict handling initially. The separate merge -draft gitlinks in [agent-github.md](agent-github.md#merges) remain a follow-up; -the agent still resolves source conflicts before publishing. - -`gh stack link` extends existing stacks; it does not remove or reorder their -members. Restructuring requires explicit GitHub stack changes and, when code -dependencies change, new source commits. Do not treat another `link` call as a -rebase or a replacement of the entire stack. - -### What we reuse and what remains - -Use `gh-stack` for GitHub's stack membership operations. The agent composes -CAOS's existing copy, merge, test and publish operations to maintain code -dependencies. This requires no separate stack manifest or persistent local -branch database. - -[`gh stack submit`, `sync` and `rebase`](https://docs.github.com/en/pull-requests/reference/stacked-prs-cli-commands) -operate on local branches and tracking state; rebasing also needs a worktree -for code and conflicts. A bare repo containing branch refs alone would not -make those workflows fit. Defer that adapter and history rewrites. - -The stack follow-up should exercise a two-layer stack in a test repository: -publish and link it, change the lower layer and propagate upward, append a -third layer, then land a lower PR and update the survivors. Include partial -failure and unavailable-stack cases. Verify the actual pinned extension with -no source checkout; version/help tests do not establish this. - -Start with that workflow and its tests. If an operation is missing, add the -smallest helper it needs after checking `gh`, `gh stack` and `gh api`. -Reordering, dropping and rebasing arbitrary layers can follow once there is -a concrete need. Landing remains a separately requested action. - -## Interfaces and compatibility - -The agent uses publication and GitHub tools for PR operations. -The TUI's publication commands and preview UI are removed. -Existing publication records remain readable. Local imports keep their TUI command. -The separate merge-draft design is deferred. - -Implementation: [server push endpoint](../rust/crates/server/src/push.rs), -[agent publication](../std/llm-step/src/publish_source.rs), -[GitHub worker](../std/github/src/main.rs), and -[publication records](../rust/crates/conversation-protocol/src/v3/records.rs). -External behavior: [Git push leases](https://git-scm.com/docs/git-push), -[GitHub stack commands](https://docs.github.com/en/pull-requests/reference/stacked-prs-cli-commands), -[gh-stack implementation](https://github.com/github/gh-stack). +Implementation: [server push endpoint](../rust/crates/server/src/push.rs), [agent publication](../std/llm-step/src/publish_source.rs), [GitHub worker](../std/github/src/main.rs), and [stack +tools](../std/llm-step/src/stack.rs). diff --git a/design/agent-stacks.md b/design/agent-stacks.md index 98d61c868..e64081218 100644 --- a/design/agent-stacks.md +++ b/design/agent-stacks.md @@ -1,6 +1,7 @@ # Agent stacks -A stack is an ordered list of source gitlinks. Git computes merges directly from objects; CAOS records the order and any paused operation. The GitHub worker manages PRs without a source checkout. +A stack is an ordered list of source gitlinks. Git computes merges directly from objects; CAOS records the order and any paused operation. Publication pushes the stack's branches directly from the +server. ```text feature/ @@ -62,3 +63,27 @@ proceeds until completion or another conflict. Draft editing commits never becom The operation survives turns and worker restarts. Completion checks the original manifest and source pointers, then replaces all stack gitlinks together. A concurrent edit causes a refusal, not an overwrite. `{"action":"abort","path":"feature"}` removes the pending attempt and leaves the sources unchanged; the draft remains in conversation history. `status` reports the pointers and pending operation. + +## Pushing branches + +`push_stack` takes `path`, an HTTPS Git `repository` URL, and optional `rewrite`: + +```json +{"path":"feature","repository":"https://github.com/owner/repo.git"} +``` + +It pushes one branch per layer, bottom to top. Branch names are the source gitlink paths (`feature/01-core`, `feature/02-ui`); the base gitlink is not pushed. Each upper commit must contain the exact +lower tip. Finish pending conflicts and restack edited lower layers before pushing. + +The tool records all source commits and observed remote heads before the first push. It uses the same `caos push-git` path as `publish_source`: the server transfers its stored objects, and an exact +expected-head lease prevents overwriting a concurrent branch update. No checkout or GitHub worker is needed. + +After rebasing published history, pass `rewrite:true`. This allows non-fast-forward updates while retaining the lease and publication checks, including `.gitignore`. An unchanged push converges on the +existing branch heads. The remote mainline need not match the stack's base just to push work; importing and rebasing onto updated mainline is a separate decision. + +Branch pushes are not one transaction. The tool records each result before proceeding, stops at the first failure or uncertain result, and returns the completed receipts and branches not attempted. +Recovery skips recorded successful pushes and retains the original commits and leases. Inspect an uncertain result before making a new call, which observes fresh remote heads. + +Git receives branches and commit ancestry. The intended stack order stays in the conversation manifest. This operation creates no PRs or GitHub stack membership; those are a separate follow-up. + +This implements linear restacking, merge propagation and branch publication. Interactive reorder/squash/drop and rebasing merge commits remain unsupported. diff --git a/std/github/src/main.rs b/std/github/src/main.rs index 9f4d2b488..5d50cff7b 100644 --- a/std/github/src/main.rs +++ b/std/github/src/main.rs @@ -47,7 +47,7 @@ fn run() -> Result<(), String> { return Err("invocation must be 64 lowercase hexadecimal characters".into()); } let request = own_args_tree()?; - let token = secret("github-token")?; + let token = secret("github-token")?.trim_end().to_owned(); let server = std::env::var("CAOS_SERVER_URL").map_err(|_| "CAOS_SERVER_URL not set")?; let mut store = GitStore::scratch("github-invocation", &server)?; let refname = format!("refs/caos/github/{invocation}"); diff --git a/std/llm-step/src/main.rs b/std/llm-step/src/main.rs index f6d1f55ae..4d3f08aaf 100644 --- a/std/llm-step/src/main.rs +++ b/std/llm-step/src/main.rs @@ -7,6 +7,7 @@ mod import_source; mod object_upload; mod progress; mod publish_source; +mod push_stack; mod source_trees; mod stack; mod subagents; @@ -1067,6 +1068,10 @@ fn drive_call( stack::execute(state, &site)?; return Ok(true); } + if existing.name == "push_stack" { + push_stack::execute(state, &site)?; + return Ok(true); + } if existing.name == "publish_source" { publish_source::execute(state, &site)?; return Ok(true); @@ -1089,7 +1094,7 @@ fn drive_call( stack::execute(state, &site)?; return Ok(true); } - if matches!(call.name.as_str(), "github") { + if call.name == "github" { return github::start(cfg, state, &site); } if call.name == subagents::SPAWN_TOOL { @@ -1102,6 +1107,10 @@ fn drive_call( harvest_agent_call(state, &site)?; return Ok(true); } + if call.name == "push_stack" { + push_stack::execute(state, &site)?; + return Ok(true); + } if call.name == "publish_source" { publish_source::execute(state, &site)?; return Ok(true); @@ -1642,7 +1651,7 @@ fn dispatch_started( call: &Call, record: &CallRecord, ) -> Result<(), String> { - if matches!(call.name.as_str(), "github") { + if call.name == "github" { return github::dispatch(request, round, call, record); } let commit = record @@ -2978,11 +2987,12 @@ fn source_tree_paths(state: &mut progress::State) -> Result, String> } fn registry(cfg: &Config) -> Result, String> { - let mut registry = vec![bash_tool(), stack::declaration()]; + let mut registry = vec![bash_tool(), stack::declaration(), push_stack::declaration()]; registry.extend(tools::declarations()); - let mut publish = with_source_tree(tools::tree_tool_declaration( - &tools::builtin_tool("publish_source", publish_source::HELP), - )); + let mut publish = with_source_tree(tools::tree_tool_declaration(&tools::builtin_tool( + "publish_source", + publish_source::HELP, + ))); publish["input_schema"]["properties"]["rewrite"]["type"] = json!("boolean"); registry.push(publish); if cfg.run_and_update_ref_image.is_some() { @@ -3673,7 +3683,10 @@ mod tests { golden_with_first("read", json!({"file-path":"files/a"})) } - pub(super) fn golden_with_first(first_name: &str, first_input: Value) -> Result { + pub(super) fn golden_with_first( + first_name: &str, + first_input: Value, + ) -> Result { let mut store = MemoryStore::new(); let root = root_with(&mut store, BTreeMap::new())?; let user = append_memory( diff --git a/std/llm-step/src/publish_source.rs b/std/llm-step/src/publish_source.rs index f63046fe7..efd90a7fc 100644 --- a/std/llm-step/src/publish_source.rs +++ b/std/llm-step/src/publish_source.rs @@ -72,49 +72,21 @@ pub(super) fn execute(state: &mut progress::State, site: &CallSite<'_>) -> Resul let pending = match pinned(&state.conversation()?, site)? { Some(record) => record, None => { - let view = state.conversation()?; - let head = match view.source_tree(&p.source_tree)? { - Some(source) => source.commit, - None => { - return site.fail(state, "publish_source requires an existing source gitlink") - } - }; - let base = view.reference_start(&p.source_tree)?; let old = match observe() { Ok(old) => old, Err(error) => return site.fail(state, &error), }; - let id = view.identity()?.id; - let descriptor = Descriptor { - source_base: base.clone(), - source_head: head.clone(), - target_base: base, - policy: if p.rewrite { "rewrite" } else { "preserve" }.into(), - implementation: "caos/server-push".into(), - commit_policy: "preserve".into(), - }; - let key = invocation(&id, site)?[..32].to_string(); - let publication = ids::publication_id( - &id, - &key, - &ids::projection_id(&descriptor.to_value())?, - &head, + let record = match plan( + &state.conversation()?, + site, + &p.source_tree, &p.repository, - &format!("refs/heads/{}", p.branch), - old.as_ref(), - )?; - let record = PublicationRecord { - id: publication, - key, - descriptor, - planned_head: head, - repository: p.repository.clone(), - refname: format!("refs/heads/{}", p.branch), - expected_old: old, - source_tree_name: p.source_tree.clone(), - status: PublicationStatus::Pending, - evidence: None, - observed: None, + &p.branch, + old, + p.rewrite, + ) { + Ok(record) => record, + Err(error) => return site.fail(state, &error), }; let Some(record) = pin(state, site, &record)? else { return Ok(()); @@ -125,17 +97,81 @@ pub(super) fn execute(state: &mut progress::State, site: &CallSite<'_>) -> Resul if pending.status != PublicationStatus::Pending { return finish(state, site, &pending, None); } - // An attempt that pinned this intent may send it, including concurrent - // attempts that joined the identical transition. The exact lease makes - // those pushes converge. Recovery never obtains a fresh source or lease. - let outcome = { + let outcome = push(&pending)?; + finish(state, site, &pending, Some(outcome)) +} + +pub(super) fn plan( + view: &Conversation<'_>, + site: &CallSite<'_>, + source_tree: &str, + repository: &str, + branch: &str, + old: Option, + rewrite: bool, +) -> Result { + let head = view + .source_tree(source_tree)? + .ok_or("publication requires an existing source gitlink")? + .commit; + let base = view.reference_start(source_tree)?; + let id = view.identity()?.id; + let descriptor = Descriptor { + source_base: base.clone(), + source_head: head.clone(), + target_base: base, + policy: if rewrite { "rewrite" } else { "preserve" }.into(), + implementation: "caos/server-push".into(), + commit_policy: "preserve".into(), + }; + let key = invocation(&id, site)?[..32].to_string(); + let publication = ids::publication_id( + &id, + &key, + &ids::projection_id(&descriptor.to_value())?, + &head, + repository, + &format!("refs/heads/{}", branch), + old.as_ref(), + )?; + Ok(PublicationRecord { + id: publication, + key, + descriptor, + planned_head: head, + repository: repository.into(), + refname: format!("refs/heads/{}", branch), + expected_old: old, + source_tree_name: source_tree.into(), + status: PublicationStatus::Pending, + evidence: None, + observed: None, + }) +} + +pub(super) fn push(pending: &PublicationRecord) -> Result { + let branch = pending + .refname + .strip_prefix("refs/heads/") + .ok_or("invalid publication branch")?; + let token_file = import_source::token_file(&pending.repository); + let token = token_file + .map(fs::read_to_string) + .transpose() + .map_err(|_| "reading GitHub token")?; + let token = token.as_deref().map(str::trim_end); + let observe = || { + git_locator::publish::read_branch(&pending.repository, branch, token) + .and_then(|h| h.map(|h| Oid::parse(&h, "remote head")).transpose()) + }; + Ok({ let mut command = std::process::Command::new("caos"); command .args([ "push-git", &pending.repository, pending.planned_head.as_str(), - &p.branch, + branch, ]) .arg(format!( "--expected={}", @@ -178,7 +214,7 @@ pub(super) fn execute(state: &mut progress::State, site: &CallSite<'_>) -> Resul value["diagnostic"].as_str().map(str::to_owned), observed, ); - reconcile(&pending, outcome, observe) + reconcile(pending, outcome, observe) } Ok(output) if output.status.code() == Some(1) => Outcome::new( PublicationStatus::Conflict, @@ -186,11 +222,10 @@ pub(super) fn execute(state: &mut progress::State, site: &CallSite<'_>) -> Resul Some(String::from_utf8_lossy(&output.stderr).trim().to_string()), None, ), - _ => recovered(&pending, observe()), + _ => recovered(pending, observe()), }, } - }; - finish(state, site, &pending, Some(outcome)) + }) } pub(super) fn invocation(conversation: &str, site: &CallSite<'_>) -> Result { diff --git a/std/llm-step/src/push_stack.rs b/std/llm-step/src/push_stack.rs new file mode 100644 index 000000000..7bea6c260 --- /dev/null +++ b/std/llm-step/src/push_stack.rs @@ -0,0 +1,398 @@ +//! Push a pinned stack through the same branch publication path as publish_source. +use super::*; +use conversation_protocol::v3::publication::Outcome; +use conversation_protocol::v3::{GitStore, PublicationRecord, PublicationStatus}; + +pub(super) fn declaration() -> Value { + json!({ + "name":"push_stack", + "description":"Push a registered stack's branches directly from the CAOS server to an HTTPS Git repository. Branch names are the source gitlink paths. Pins all commits and remote-head leases before pushing bottom to top. Does not create PRs or GitHub stack membership, and needs no source checkout or GitHub worker. Finish or abort pending conflicts and restack edited lower layers first. Set rewrite=true for intentionally rebased history; exact leases still reject concurrent branch changes. Pushes are not atomic across branches: inspect the returned receipts after partial failure. Recovery keeps the original commits and leases and skips recorded successful pushes.", + "input_schema":{"type":"object","additionalProperties":false,"properties":{ + "path":{"type":"string","description":"Registered stack directory."}, + "repository":{"type":"string","description":"HTTPS Git repository URL, without credentials."}, + "rewrite":{"type":"boolean","description":"Allow intentionally rewritten history with an exact remote-head lease."} + },"required":["path","repository"]} + }) +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Parameters { + path: String, + repository: String, + #[serde(default)] + rewrite: bool, +} + +fn payload(site: &CallSite<'_>) -> String { + format!( + "{}/publications.json", + paths::call_payload_dir(site.request.as_str(), site.round, &site.call.id) + ) +} + +fn saved( + view: &Conversation<'_>, + site: &CallSite<'_>, +) -> Result>, String> { + if view + .tool(site.request, site.round, &site.call.id)? + .is_none() + { + return Ok(None); + } + serde_json::from_slice(&view.payload(&payload(site))?) + .map(Some) + .map_err(|_| "invalid pinned stack publication".into()) +} + +fn plan(state: &progress::State, site: &CallSite<'_>) -> Result, String> { + let p: Parameters = serde_json::from_value(site.call.input.clone()) + .map_err(|e| format!("invalid push_stack arguments: {e}"))?; + git_locator::import::remote(&p.repository)?; + let server = std::env::var("CAOS_SERVER_URL").map_err(|_| "CAOS_SERVER_URL not set")?; + let store = GitStore::scratch_partial(&fresh_name("push-stack-objects"), &server)?; + let view = state.conversation()?; + let branches = stack::push_sources(&view, &p.path, &store)?; + let token = import_source::token_file(&p.repository) + .map(fs::read_to_string) + .transpose() + .map_err(|_| "reading GitHub token")?; + branches + .into_iter() + .map(|branch| { + let old = git_locator::publish::read_branch( + &p.repository, + &branch, + token.as_deref().map(str::trim_end), + )? + .map(|h| Oid::parse(&h, "remote head")) + .transpose()?; + publish_source::plan(&view, site, &branch, &p.repository, &branch, old, p.rewrite) + }) + .collect() +} + +pub(super) fn execute(state: &mut progress::State, site: &CallSite<'_>) -> Result<(), String> { + if state + .conversation()? + .tool(site.request, site.round, &site.call.id)? + .is_some_and(|r| r.is_terminal()) + { + return Ok(()); + } + let plans = match saved(&state.conversation()?, site)? { + Some(plans) => plans, + None => match plan(state, site) { + Ok(plans) => plans, + Err(error) => return site.fail(state, &error), + }, + }; + let Some(plans) = pin(state, site, plans)? else { + return Ok(()); + }; + advance(state, site, &plans, publish_source::push) +} + +fn pin( + state: &mut progress::State, + site: &CallSite<'_>, + plans: Vec, +) -> Result>, String> { + for _ in 0..32 { + state.reload()?; + let view = state.conversation()?; + if view + .tool(site.request, site.round, &site.call.id)? + .is_some_and(|r| r.is_terminal()) + { + return Ok(None); + } + if let Some(plans) = saved(&view, site)? { + return Ok(Some(plans)); + } + let mut record = site.stub(None); + record.status = CallStatus::Started; + let expected = state.head().clone(); + if matches!( + state.try_append_at( + &expected, + Transition::ToolStart { + record, + payloads: vec![( + "publications.json".into(), + canonical_payload_bytes(&json!(plans))? + )], + } + )?, + progress::TryAppend::Appended(_) + ) { + return Ok(Some(plans)); + } + } + Err("conversation kept moving while pinning stack publication".into()) +} + +fn advance( + state: &mut progress::State, + site: &CallSite<'_>, + plans: &[PublicationRecord], + mut push: impl FnMut(&PublicationRecord) -> Result, +) -> Result<(), String> { + let mut receipts = Vec::new(); + for plan in plans { + state.reload()?; + let record = match state.conversation()?.publication(&plan.id)? { + Some(record) => record, + None => { + state.append(Transition::PublicationPending { + record: plan.clone(), + })?; + plan.clone() + } + }; + let record = if record.status == PublicationStatus::Pending { + let outcome = push(&record)?; + retain(state, &record, outcome)? + } else { + record + }; + let complete = record.status == PublicationStatus::Complete; + receipts.push(record); + if !complete { + break; + } + } + let complete = receipts.len() == plans.len() + && receipts + .iter() + .all(|r| r.status == PublicationStatus::Complete); + let remaining: Vec<_> = plans[receipts.len()..].iter().map(|r| &r.refname).collect(); + let result = json!({"status":if complete {"complete"} else {"partial"}, "branches":receipts, "not_attempted":remaining}); + let block = result_block(&site.call.id, &result.to_string(), !complete); + let stub = site.stub(None); + let record = completed_record( + &stub, + ToolResult::Complete { + observation: observation_path(&stub), + proposal: None, + }, + None, + ); + let transition = tool_complete_transition(record, &block, Vec::new())?; + for _ in 0..32 { + state.reload()?; + if state + .conversation()? + .tool(site.request, site.round, &site.call.id)? + .is_some_and(|r| r.is_terminal()) + { + return Ok(()); + } + let head = state.head().clone(); + if matches!( + state.try_append_at(&head, transition.clone())?, + progress::TryAppend::Appended(_) + ) { + return Ok(()); + } + } + Err("conversation kept moving while completing stack publication".into()) +} + +fn retain( + state: &mut progress::State, + pending: &PublicationRecord, + outcome: Outcome, +) -> Result { + for _ in 0..32 { + state.reload()?; + let record = state + .conversation()? + .publication(&pending.id)? + .ok_or("publication disappeared")?; + if record.status != PublicationStatus::Pending { + return Ok(record); + } + let head = state.head().clone(); + if matches!( + state.try_append_at( + &head, + Transition::PublicationTerminal { + publication: record.id, + status: outcome.status, + evidence: outcome.evidence.clone(), + observed: outcome.observed.clone(), + } + )?, + progress::TryAppend::Appended(_) + ) { + return state + .conversation()? + .publication(&pending.id)? + .ok_or("publication disappeared".into()); + } + } + Err("conversation kept moving while saving branch publication".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{golden_with_first, ImportStore}; + + fn fixture() -> (progress::State, Oid, Call, String) { + let args = json!({"path":"feature", "repository":"https://example.com/repo.git"}); + let golden = golden_with_first("push_stack", args.clone()).unwrap(); + let request = golden.request.clone(); + let declaration = Conversation::open(&golden.store, &golden.head) + .unwrap() + .transcript_entry(1) + .unwrap() + .unwrap() + .1 + .message_id; + let call = Call { + id: "first".into(), + name: "push_stack".into(), + input: args, + }; + let store = ImportStore { + objects: golden.store, + head: golden.head.clone(), + race: None, + lost_ack: true, + }; + let mut state = progress::State::from_store( + store, + "refs/conversations/conversation/head".into(), + golden.head, + ) + .unwrap(); + for (name, c) in [("a", 'a'), ("b", 'b'), ("c", 'c')] { + state + .append(Transition::reference( + format!("feature/{name}"), + Some(Oid::parse(&c.to_string().repeat(40), "head").unwrap()), + )) + .unwrap(); + } + (state, request, call, declaration) + } + + fn plans(state: &progress::State, site: &CallSite<'_>) -> Vec { + ["a", "b", "c"] + .into_iter() + .map(|name| { + let path = format!("feature/{name}"); + publish_source::plan( + &state.conversation().unwrap(), + site, + &path, + "https://example.com/repo.git", + &path, + Some(Oid::parse(&"d".repeat(40), "old").unwrap()), + true, + ) + .unwrap() + }) + .collect() + } + + #[test] + fn restart_reuses_pinned_stack_and_skips_completed_branches() { + let (mut state, request, call, declaration) = fixture(); + let site = CallSite::at(&request, 0, &call, &declaration); + let original = plans(&state, &site); + let pinned = pin(&mut state, &site, original.clone()).unwrap().unwrap(); + let mut calls = 0; + let failed = advance(&mut state, &site, &pinned, |record| { + calls += 1; + if calls == 2 { + return Err("worker stopped before receiving a result".into()); + } + Ok(Outcome::new( + PublicationStatus::Complete, + "push-success", + None, + Some(record.planned_head.clone()), + )) + }); + assert!(failed.is_err()); + state + .append(Transition::reference( + "feature/b".into(), + Some(Oid::parse(&"e".repeat(40), "edit").unwrap()), + )) + .unwrap(); + let mut newer = plans(&state, &site); + newer[1].expected_old = None; + let recovered = pin(&mut state, &site, newer).unwrap().unwrap(); + assert_eq!(recovered, original); + let mut pushed = Vec::new(); + advance(&mut state, &site, &recovered, |record| { + pushed.push(record.refname.clone()); + assert_eq!(record.expected_old, original[1].expected_old); + Ok(Outcome::new( + PublicationStatus::Complete, + "ref-converged", + None, + Some(record.planned_head.clone()), + )) + }) + .unwrap(); + assert_eq!(pushed, ["refs/heads/feature/b", "refs/heads/feature/c"]); + assert!(pin(&mut state, &site, original).unwrap().is_none()); + validate_spine(state.store(), state.head(), &mut HashSet::new()).unwrap(); + } + + #[test] + fn failed_layer_reports_partial_progress_and_does_not_push_upper_layers() { + for status in [PublicationStatus::Conflict, PublicationStatus::Uncertain] { + let (mut state, request, call, declaration) = fixture(); + let site = CallSite::at(&request, 0, &call, &declaration); + let plans = plans(&state, &site); + pin(&mut state, &site, plans.clone()).unwrap().unwrap(); + let mut calls = 0; + advance(&mut state, &site, &plans, |record| { + calls += 1; + Ok(if calls == 1 { + Outcome::new( + PublicationStatus::Complete, + "push-success", + None, + Some(record.planned_head.clone()), + ) + } else { + Outcome::new( + status, + "lease-rejected", + Some("remote changed".into()), + None, + ) + }) + }) + .unwrap(); + assert_eq!(calls, 2); + let view = state.conversation().unwrap(); + assert_eq!( + view.publication(&plans[0].id).unwrap().unwrap().status, + PublicationStatus::Complete + ); + assert_eq!( + view.publication(&plans[1].id).unwrap().unwrap().status, + status + ); + assert!(view.publication(&plans[2].id).unwrap().is_none()); + let record = view.tool(&request, 0, &call.id).unwrap().unwrap(); + let observation: Value = + serde_json::from_slice(&view.payload(&observation_path(&record)).unwrap()).unwrap(); + assert_eq!(observation["is_error"], true); + let receipt: Value = + serde_json::from_str(observation["content"][0]["text"].as_str().unwrap()).unwrap(); + assert_eq!(receipt["status"], "partial"); + assert_eq!(receipt["not_attempted"], json!(["refs/heads/feature/c"])); + validate_spine(state.store(), state.head(), &mut HashSet::new()).unwrap(); + } + } +} diff --git a/std/llm-step/src/source_trees.rs b/std/llm-step/src/source_trees.rs index 8a5357306..a2a17beef 100644 --- a/std/llm-step/src/source_trees.rs +++ b/std/llm-step/src/source_trees.rs @@ -6,6 +6,7 @@ Stack workflow: - Register an ordered set of existing source gitlinks with stack(action="create", path="feature", base="00-base", layers=["01-core","02-ui"]). Keep the base as an imported snapshot. - After a lower-layer edit or an import of updated origin/main, use stack(action="rebase", path="feature", onto=""). Use method="merge" for histories containing merge commits. - A stack conflict leaves the source pointers unchanged. Read feature/restack/operation.json and edit/test feature/restack/work. Then stack(action="continue", path="feature", resolved=true) explicitly asserts that ALL reported conflicts are resolved, including structural ones. Do not create .caos/conflicts. Abort leaves sources unchanged. +- To push a registered stack, use push_stack with path and an HTTPS repository URL. It pushes the source gitlink paths as branch names and creates no PRs or GitHub stack membership. Use rewrite=true only for intentional rebased-history updates. After uncertainty inspect the remote before a new call. Conversation filesystem: @@ -19,6 +20,7 @@ Conversation filesystem: - Before integrating a publication base, compare the intended feature change with the full source-versus-destination difference. Merging upstream preserves all existing branch changes; rebasing a whole branch may replay inherited changes too. Transplanting only the requested edit onto a new base is a separate operation. If a small task would publish unrelated inherited work, explain that scope and ask whether to retain the branch or transplant the edit. Do not claim that targeting an older upstream isolates the edit. Preserve existing snapshots when making a new starting point. - For legacy non-stack merge operations, resolve source-tree conflicts by editing affected files and clearing their ledger entries. Saving an edited source tree removes an empty .caos/conflicts ledger and prunes its .caos directory if empty. Unresolved entries and other metadata stay intact. Do not recreate a removed ledger to register resolution. This source-tree ledger is distinct from protected conversation-root .caos. - Preparing, delegating, merging, testing, and organizing a PR stack requires no publishing destination. Preserve the imported starting commit at 00-base. Finish the requested work without asking for repository publication settings. +- When publication is requested, use publish_source with an explicit gitlink, HTTPS repository and branch. Inspect the complete PR diff, incorporate the chosen base and prior stack boundary, resolve conflicts and run checks first. Use push_stack for registered stacks. PR metadata changes are separate GitHub operations. No local branch database is needed. Import provenance can identify the repository and default base; ask only when ambiguous. - If a tool fails, report the observed error and uncertainty; do not invent storage behavior or claim success from a failed check. Use available repository tools for relevant tests; describe a specific missing capability rather than assuming workers cannot run tests or reach a server. - Git tools require the target commit-entry path explicitly. Invoke repository tools with run_tool at a conversation-relative path. UI selection never changes your execution context. @@ -26,7 +28,9 @@ Conversation filesystem: - Use import_source(source="https://github.com/owner/repo.git", revision="main", into="imports/repo/main-2") for remote code. Omit revision for the default branch. Public imports need no token; private imports use the granted github-token secret. Each new call observes the remote; retries of a persisted call keep its pinned commit. Choose an unused path and preserve the imported snapshot. The tool returns a full commit hash and records provenance in its .source.json sibling. It does not merge code. - For origin/main, read the selected source's import provenance, use that repository URL and revision main with import_source, then use stack rebase for a registered stack, or merge it into an individual source tree. If provenance is ambiguous, use an explicit repository. Remote names are not local ref snapshots. -- When a lower layer changes, restack the registered stack, test, and resubmit. After a lower PR lands, import actual mainline (squash/rebase may change hashes), integrate it and inspect the surviving diff before retargeting. gh stack submit/sync/rebase need local branches; use the CAOS stack tools instead. Rebased publication requires explicit rewrite=true and retains an exact remote-head lease. Publishing does not authorize landing PRs. +- Push stack layers bottom to top with push_stack. Each upper source commit must include the exact published lower commit. The tool pins commits and remote-head leases, records branch receipts, and stops after a failure. Never roll back successful pushes after a later branch fails. +- When a lower layer changes, restack the registered stack, test, and push again. Rebased publication requires explicit rewrite=true and retains an exact remote-head lease. Pushing a stack does not create PRs or GitHub stack membership. + Client actions (give these instructions to the user, not to bash or run_tool): - When a task needs a client action, give the exact TUI command or keys and concrete conversation path. Do not just say "ask to import", "trigger an import", or "publish it". Use supplied paths/URLs; ask only for missing information. Do not claim a client operation succeeded without its result. Subagents report client needs to their parent. diff --git a/std/llm-step/src/stack.rs b/std/llm-step/src/stack.rs index d307659c9..06ecd2406 100644 --- a/std/llm-step/src/stack.rs +++ b/std/llm-step/src/stack.rs @@ -452,7 +452,7 @@ fn complete( state: &mut progress::State, site: &CallSite<'_>, guard: &[Expected], - files: Vec<(String, Option<(Mode, Vec)>)>, + files: FileEdits, result: Value, ) -> Result<(), String> { for _ in 0..32 { @@ -532,6 +532,40 @@ fn complete( Err("conversation kept moving while saving stack result".into()) } +/// Source paths in stack order, checked before any publication is pinned. +pub(super) fn push_sources( + view: &Conversation<'_>, + path: &str, + store: &GitStore, +) -> Result, String> { + paths::validate_source_tree_name(path)?; + if view.snapshot().exists(&format!("{path}/restack"))? { + return Err("finish or abort the pending stack update before pushing".into()); + } + let manifest: Manifest = read_json(view, &manifest_path(path))?; + if manifest.layers.is_empty() { + return Err("a stack needs at least one layer".into()); + } + let mut lower = source(view, &child(path, &manifest.base)?)?; + let mut branches = Vec::new(); + for boundary in &manifest.layers { + let branch = child(path, &boundary.name)?; + conversation_protocol::v3::source_trees::validate_branch(&branch)?; + if branch.starts_with("refs/") { + return Err("branch names must omit refs/heads/".into()); + } + let commit = source(view, &branch)?; + if boundary.base != lower || !store.is_ancestor(&lower, &commit)? { + return Err(format!( + "{branch} does not contain the current lower layer; restack before pushing" + )); + } + branches.push(branch); + lower = commit; + } + Ok(branches) +} + #[cfg(test)] mod tests { use super::*; diff --git a/std/llm-step/src/tools.rs b/std/llm-step/src/tools.rs index 21a171a25..aa0e395de 100644 --- a/std/llm-step/src/tools.rs +++ b/std/llm-step/src/tools.rs @@ -112,6 +112,7 @@ pub fn grep_declaration() -> Value { /// are standard, not project-defined. const RESERVED_TOOLS: &[&str] = &[ "stack", + "push_stack", "import_source", "publish_source", "github", From 176a511d861687cc992602270a4db0563756cc64 Mon Sep 17 00:00:00 2001 From: Nishad <133812901+nishu-builder@users.noreply.github.com> Date: Sat, 19 Sep 2026 23:24:59 +0000 Subject: [PATCH 2/2] Share publication recovery between branch and stack pushes Incorporate the simplifications from #266. Both publication tools retain remote outcomes through the same durable record path, and unreadable successful command results are reconciled against the remote branch. --- std/llm-step/src/main.rs | 44 ++++++++++----- std/llm-step/src/publish_source.rs | 89 ++++++++++++++++++++---------- std/llm-step/src/push_stack.rs | 38 +------------ 3 files changed, 91 insertions(+), 80 deletions(-) diff --git a/std/llm-step/src/main.rs b/std/llm-step/src/main.rs index 4d3f08aaf..7c894d609 100644 --- a/std/llm-step/src/main.rs +++ b/std/llm-step/src/main.rs @@ -4441,15 +4441,29 @@ mod tests { .unwrap() .unwrap(); assert_eq!(recovered, pending); + for malformed in [ + b"{\"status\":".as_slice(), + br#"{"status":"unknown","kind":"push-success","observed":null}"#, + br#"{"status":"complete","observed":null}"#, + br#"{"status":"complete","kind":"push-success","observed":"invalid-oid"}"#, + ] { + for (observed, expected_status) in [ + (Ok(Some(commit.clone())), PublicationStatus::Complete), + (Ok(Some(old.clone())), PublicationStatus::Uncertain), + (Ok(Some(newer.clone())), PublicationStatus::Conflict), + ( + Err("remote unavailable".into()), + PublicationStatus::Uncertain, + ), + ] { + let outcome = publish_source::reconcile(&pending, malformed, || observed); + assert_eq!(outcome.status, expected_status); + } + } let outcome = if rejected { publish_source::reconcile( &recovered, - conversation_protocol::v3::publication::Outcome::new( - PublicationStatus::Conflict, - "validation-rejected", - Some("Source contains ignored files".into()), - None, - ), + br#"{"status":"conflict","kind":"validation-rejected","diagnostic":"Source contains ignored files","observed":null}"#, || panic!("validation rejection must not observe the remote"), ) } else { @@ -4457,16 +4471,20 @@ mod tests { // rejected old value even though its first push succeeded. publish_source::reconcile( &recovered, - conversation_protocol::v3::publication::Outcome::new( - PublicationStatus::Conflict, - "lease-rejected", - None, - None, - ), + br#"{"status":"conflict","kind":"lease-rejected","observed":null}"#, || Ok(Some(commit.clone())), ) }; - publish_source::finish(&mut state, &site, &pending, Some(outcome)).unwrap(); + // Both tools retain the remote result before completing the call. + // A restart in between must use that result without pushing again. + let outcome = if rejected { + publish_source::retain(&mut state, &pending, outcome).unwrap(); + state.reload().unwrap(); + None + } else { + Some(outcome) + }; + publish_source::finish(&mut state, &site, &pending, outcome).unwrap(); let view = state.conversation().unwrap(); assert_eq!( view.source_tree("feature/lower").unwrap().unwrap().commit, diff --git a/std/llm-step/src/publish_source.rs b/std/llm-step/src/publish_source.rs index efd90a7fc..0df5a6511 100644 --- a/std/llm-step/src/publish_source.rs +++ b/std/llm-step/src/publish_source.rs @@ -200,21 +200,7 @@ pub(super) fn push(pending: &PublicationRecord) -> Result { ), Ok(child) => match child.wait_with_output() { Ok(output) if output.status.success() => { - let value: Value = serde_json::from_slice(&output.stdout) - .map_err(|_| "invalid push-git result")?; - let status: PublicationStatus = serde_json::from_value(value["status"].clone()) - .map_err(|_| "invalid push status")?; - let observed: Option = serde_json::from_value(value["observed"].clone()) - .map_err(|_| "invalid remote head")?; - let outcome = Outcome::new( - status, - value["kind"] - .as_str() - .ok_or("missing publication evidence")?, - value["diagnostic"].as_str().map(str::to_owned), - observed, - ); - reconcile(pending, outcome, observe) + reconcile(pending, &output.stdout, observe) } Ok(output) if output.status.code() == Some(1) => Outcome::new( PublicationStatus::Conflict, @@ -228,6 +214,16 @@ pub(super) fn push(pending: &PublicationRecord) -> Result { }) } +fn parse_outcome(bytes: &[u8]) -> Option { + let value: Value = serde_json::from_slice(bytes).ok()?; + Some(Outcome::new( + serde_json::from_value(value["status"].clone()).ok()?, + value["kind"].as_str()?, + value["diagnostic"].as_str().map(str::to_owned), + serde_json::from_value(value["observed"].clone()).ok()?, + )) +} + pub(super) fn invocation(conversation: &str, site: &CallSite<'_>) -> Result { ids::protocol_id( "external-tool", @@ -239,9 +235,13 @@ pub(super) fn invocation(conversation: &str, site: &CallSite<'_>) -> Result Result, String>, ) -> Outcome { + let Some(outcome) = parse_outcome(output) else { + // The push may have completed despite an unreadable command result. + return recovered(pending, observe()); + }; // A local/server validation refusal means no push was attempted. A remote // head that already matches must not hide the rejection or its diagnostic. if outcome.status == PublicationStatus::Complete @@ -321,12 +321,51 @@ pub(super) fn pin( Err("conversation kept moving while pinning publication".into()) } +pub(super) fn retain( + state: &mut progress::State, + pending: &PublicationRecord, + outcome: Outcome, +) -> Result { + for _ in 0..32 { + state.reload()?; + let record = state + .conversation()? + .publication(&pending.id)? + .ok_or("publication disappeared")?; + if record.status != PublicationStatus::Pending { + return Ok(record); + } + let head = state.head().clone(); + if matches!( + state.try_append_at( + &head, + Transition::PublicationTerminal { + publication: record.id, + status: outcome.status, + evidence: outcome.evidence.clone(), + observed: outcome.observed.clone(), + } + )?, + progress::TryAppend::Appended(_) + ) { + return state + .conversation()? + .publication(&pending.id)? + .ok_or("publication disappeared".into()); + } + } + Err("conversation kept moving while saving publication".into()) +} + pub(super) fn finish( state: &mut progress::State, site: &CallSite<'_>, pending: &PublicationRecord, outcome: Option, ) -> Result<(), String> { + if let Some(outcome) = outcome { + retain(state, pending, outcome)?; + } for _ in 0..32 { state.reload()?; let view = state.conversation()?; @@ -336,21 +375,11 @@ pub(super) fn finish( { return Ok(()); } - let mut record = view + let record = view .publication(&pending.id)? .ok_or("publication disappeared")?; - let mut transitions = Vec::new(); if record.status == PublicationStatus::Pending { - let out = outcome.as_ref().ok_or("missing publication outcome")?; - record.status = out.status; - record.evidence = Some(out.evidence.clone()); - record.observed = out.observed.clone(); - transitions.push(Transition::PublicationTerminal { - publication: record.id.clone(), - status: out.status, - evidence: out.evidence.clone(), - observed: out.observed.clone(), - }); + return Err("missing publication outcome".into()); } let text = serde_json::to_string(&record).map_err(|e| e.to_string())?; let block = result_block( @@ -367,10 +396,10 @@ pub(super) fn finish( }, None, ); - transitions.push(tool_complete_transition(tool, &block, Vec::new())?); + let transition = tool_complete_transition(tool, &block, Vec::new())?; let expected = state.head().clone(); if matches!( - state.try_append_many_at(&expected, transitions)?, + state.try_append_at(&expected, transition)?, progress::TryAppend::Appended(_) ) { return Ok(()); diff --git a/std/llm-step/src/push_stack.rs b/std/llm-step/src/push_stack.rs index 7bea6c260..293d585b7 100644 --- a/std/llm-step/src/push_stack.rs +++ b/std/llm-step/src/push_stack.rs @@ -153,7 +153,7 @@ fn advance( }; let record = if record.status == PublicationStatus::Pending { let outcome = push(&record)?; - retain(state, &record, outcome)? + publish_source::retain(state, &record, outcome)? } else { record }; @@ -200,42 +200,6 @@ fn advance( Err("conversation kept moving while completing stack publication".into()) } -fn retain( - state: &mut progress::State, - pending: &PublicationRecord, - outcome: Outcome, -) -> Result { - for _ in 0..32 { - state.reload()?; - let record = state - .conversation()? - .publication(&pending.id)? - .ok_or("publication disappeared")?; - if record.status != PublicationStatus::Pending { - return Ok(record); - } - let head = state.head().clone(); - if matches!( - state.try_append_at( - &head, - Transition::PublicationTerminal { - publication: record.id, - status: outcome.status, - evidence: outcome.evidence.clone(), - observed: outcome.observed.clone(), - } - )?, - progress::TryAppend::Appended(_) - ) { - return state - .conversation()? - .publication(&pending.id)? - .ok_or("publication disappeared".into()); - } - } - Err("conversation kept moving while saving branch publication".into()) -} - #[cfg(test)] mod tests { use super::*;