From 36e3388e0eba735aba171f11363adcc7da19f42e Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 09:04:50 +1300 Subject: [PATCH 1/8] feat: write-env-state records a deploy in argocd-state Sets tag and revision in /.yaml and pushes with the release-train App's token, retrying when another deploy lands first. The file has to exist already. Co-Authored-By: Claude Fable 5.1 --- scripts/write_env_state.py | 116 +++++++++++++++++++++++ tests/test_write_env_state.py | 168 ++++++++++++++++++++++++++++++++++ write-env-state/action.yml | 72 +++++++++++++++ 3 files changed, 356 insertions(+) create mode 100644 scripts/write_env_state.py create mode 100644 tests/test_write_env_state.py create mode 100644 write-env-state/action.yml diff --git a/scripts/write_env_state.py b/scripts/write_env_state.py new file mode 100644 index 0000000..65f414c --- /dev/null +++ b/scripts/write_env_state.py @@ -0,0 +1,116 @@ +"""Record a deploy in mindsdb/argocd-state. + +Argo CD deploys each long-lived environment from one file per service in that +repository, ``/.yaml`` (its README has the keys). After a build, +CI sets two of them, ``tag`` and ``revision``, and Argo CD does the rest. Every +repository's deploy job does the same thing, so it is written once here and +the ``write-env-state`` action calls it. + +The file is edited line by line, not parsed and dumped, so the keys set when +the service was added keep their order and comments. It has to exist already: +adding a service means choosing its chart path and release name, which is a +reviewed change in argocd-state, not something a deploy creates on the way. + +Two deploys can land at once, so the push is retried: each attempt starts from +the branch as it is on the remote, applies the edit again and pushes. A push +the remote refuses because the branch moved is the only thing retried. + + write_env_state.py --repo-dir --env dev --service scratchpad-controller \ + --tag development-head- --revision +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +# The same shapes argocd-state's validate.py accepts. +LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") +TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +class StateError(Exception): + """A reason the deploy cannot be recorded, in words for the job log.""" + + +def check(env: str, service: str, tag: str, revision: str) -> None: + for name, value, pattern, shape in ( + ("env", env, LABEL, "a namespace name"), + ("service", service, LABEL, "a service name"), + ("tag", tag, TAG, "an image tag"), + ("revision", revision, COMMIT, "a 40-character commit sha"), + ): + if not pattern.match(value): + raise StateError(f"{name} {value!r} is not {shape}") + + +def updated_text(text: str, tag: str, revision: str) -> str: + """The state file with its tag and revision lines replaced, everything else as it was.""" + out = [] + seen = set() + for line in text.splitlines(keepends=True): + key = line.split(":", 1)[0] if ":" in line else None + if key in ("tag", "revision") and not line.startswith((" ", "#")): + seen.add(key) + value = tag if key == "tag" else revision + line = f"{key}: {value}\n" + out.append(line) + missing = {"tag", "revision"} - seen + if missing: + raise StateError(f"the file has no {' or '.join(sorted(missing))} line") + return "".join(out) + + +def git(repo_dir: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True, check=check) + + +def record(repo_dir: Path, branch: str, env: str, service: str, tag: str, revision: str, attempts: int = 5) -> str: + """Commit and push the edit; the message says what happened. Raises StateError when it can't.""" + check(env, service, tag, revision) + path = Path(env) / f"{service}.yaml" + for attempt in range(1, attempts + 1): + git(repo_dir, "fetch", "--quiet", "origin", branch) + git(repo_dir, "reset", "--quiet", "--hard", f"origin/{branch}") + file = repo_dir / path + if not file.is_file(): + raise StateError(f"{path} does not exist in argocd-state; adding a service is a reviewed change there, not a deploy") + text = file.read_text(encoding="utf-8") + new = updated_text(text, tag, revision) + if new == text: + return f"{path} already records {tag} at {revision}" + file.write_text(new, encoding="utf-8") + git(repo_dir, "add", str(path)) + git(repo_dir, "commit", "--quiet", "-m", f"{env}/{service}: {tag}") + push = git(repo_dir, "push", "--quiet", "origin", f"HEAD:{branch}", check=False) + if push.returncode == 0: + return f"{path}: tag {tag}, revision {revision} (attempt {attempt})" + if "rejected" not in push.stderr and "fetch first" not in push.stderr and "non-fast-forward" not in push.stderr: + raise StateError(f"push to {branch} failed: {push.stderr.strip()}") + raise StateError(f"{branch} kept moving; gave up after {attempts} attempts") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument("--repo-dir", type=Path, required=True, help="a clone of argocd-state with origin set") + parser.add_argument("--branch", default="main") + parser.add_argument("--env", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--attempts", type=int, default=5) + args = parser.parse_args(argv) + try: + print(record(args.repo_dir, args.branch, args.env, args.service, args.tag, args.revision, args.attempts)) + except StateError as error: + print(f"::error::{error}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_write_env_state.py b/tests/test_write_env_state.py new file mode 100644 index 0000000..71f93a9 --- /dev/null +++ b/tests/test_write_env_state.py @@ -0,0 +1,168 @@ +"""scripts/write_env_state.py against a local bare repository standing in for argocd-state.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import write_env_state # noqa: E402 + +SHA = "8b69e1a864b0f730e4552cb5616f92a9578288fa" +OTHER_SHA = "d90ad897ac744f54b0eeb69f21deefdceeda2366" +STATE = ( + "repo: https://github.com/mindsdb/scratchpad-controller.git\n" + "chart: deployment/scratchpad-controller\n" + "release: scratchpad-controller\n" + "# the image CI last built\n" + f"tag: development-head-{OTHER_SHA}\n" + f"revision: {OTHER_SHA}\n" +) + + +def git(cwd, *args): + return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, check=True).stdout + + +@pytest.fixture +def remote(tmp_path): + """A bare argocd-state with main holding dev/scratchpad-controller.yaml, and a clone of it.""" + bare = tmp_path / "argocd-state.git" + subprocess.run(["git", "init", "--quiet", "--bare", "--initial-branch=main", str(bare)], check=True) + seed = tmp_path / "seed" + subprocess.run(["git", "clone", "--quiet", str(bare), str(seed)], check=True) + git(seed, "config", "user.name", "seed") + git(seed, "config", "user.email", "seed@example.com") + git(seed, "checkout", "--quiet", "-b", "main") + (seed / "dev").mkdir() + (seed / "dev" / "scratchpad-controller.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "seed") + git(seed, "push", "--quiet", "origin", "main") + clone = tmp_path / "clone" + subprocess.run(["git", "clone", "--quiet", str(bare), str(clone)], check=True) + git(clone, "config", "user.name", "bot") + git(clone, "config", "user.email", "bot@example.com") + return bare, seed, clone + + +def main_file(bare, path="dev/scratchpad-controller.yaml"): + return subprocess.run(["git", "-C", str(bare), "show", f"main:{path}"], capture_output=True, text=True, check=True).stdout + + +class TestUpdatedText: + def test_replaces_only_the_two_lines(self): + out = write_env_state.updated_text(STATE, f"development-head-{SHA}", SHA) + assert out == STATE.replace(OTHER_SHA, SHA) + assert "# the image CI last built\n" in out + + def test_a_file_without_both_lines_is_refused(self): + with pytest.raises(write_env_state.StateError, match="no revision line"): + write_env_state.updated_text("tag: x\n", "y", SHA) + with pytest.raises(write_env_state.StateError, match="no revision or tag line"): + write_env_state.updated_text("repo: x\n", "y", SHA) + + +class TestCheck: + @pytest.mark.parametrize("field, value", [ + ("env", "Dev"), ("service", "my_service"), ("tag", "-x"), ("tag", "a b"), + ("revision", "main"), ("revision", SHA[:7]), ("revision", SHA.upper()), + ]) + def test_each_value_has_a_shape(self, field, value): + values = {"env": "dev", "service": "scratchpad-controller", "tag": "development-head-" + SHA, "revision": SHA} + values[field] = value + with pytest.raises(write_env_state.StateError, match=f"{field} "): + write_env_state.check(**values) + + +class TestRecord: + def test_commits_and_pushes_the_edit(self, remote): + bare, _, clone = remote + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + assert message.startswith("dev/scratchpad-controller.yaml: tag development-head-") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + assert git(bare, "log", "-1", "--format=%s", "main") == f"dev/scratchpad-controller: development-head-{SHA}\n" + + def test_nothing_to_do_pushes_nothing(self, remote): + bare, _, clone = remote + before = git(bare, "rev-parse", "main") + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{OTHER_SHA}", OTHER_SHA) + assert message.endswith(f"already records development-head-{OTHER_SHA} at {OTHER_SHA}") + assert git(bare, "rev-parse", "main") == before + + def test_a_missing_file_is_refused_not_created(self, remote): + bare, _, clone = remote + with pytest.raises(write_env_state.StateError, match="dev/auth.yaml does not exist in argocd-state; adding a service is a reviewed change"): + write_env_state.record(clone, "main", "dev", "auth", "x", SHA) + assert "dev/auth.yaml" not in git(bare, "ls-tree", "-r", "--name-only", "main") + + def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): + # Another service deployed after this clone was made: both edits land. + bare, seed, clone = remote + (seed / "dev" / "auth.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "dev/auth: x") + git(seed, "push", "--quiet", "origin", "main") + write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + + def test_retries_when_the_branch_moves_during_the_push(self, remote, monkeypatch): + # The remote gains a commit between this attempt's fetch and its push, as a + # concurrent deploy does; the push is rejected and the next attempt lands on top. + bare, seed, clone = remote + real_git = write_env_state.git + raced = [] + + def racing_git(repo_dir, *args, check=True): + if args[0] == "push" and not raced: + raced.append(True) + (seed / "dev" / "auth.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "dev/auth: x") + git(seed, "push", "--quiet", "origin", "main") + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", racing_git) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + assert message.endswith("(attempt 2)") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") + + def test_gives_up_after_the_attempts(self, remote, monkeypatch): + bare, seed, clone = remote + real_git = write_env_state.git + pushes = [] + + def always_racing_git(repo_dir, *args, check=True): + if args[0] == "push": + pushes.append(True) + (seed / "dev" / f"auth{len(pushes)}.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "race") + git(seed, "push", "--quiet", "origin", "main") + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", always_racing_git) + with pytest.raises(write_env_state.StateError, match="gave up after 2 attempts"): + write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA, attempts=2) + assert len(pushes) == 2 + + +class TestMain: + def test_a_refusal_is_an_error_annotation_and_exit_1(self, remote, capsys): + _, _, clone = remote + rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--tag", "x", "--revision", "main"]) + assert rc == 1 + assert capsys.readouterr().out.startswith("::error::revision 'main' is not a 40-character commit sha") + + +class TestAction: + def test_the_action_calls_the_script_with_every_input(self): + import yaml + action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) + assert set(action["inputs"]) == {"env", "service", "tag", "revision", "token", "repository", "branch"} + run = action["runs"]["steps"][-1]["run"] + for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--tag "$TAG"', '--revision "$REVISION"'): + assert flag in run diff --git a/write-env-state/action.yml b/write-env-state/action.yml new file mode 100644 index 0000000..44f15d1 --- /dev/null +++ b/write-env-state/action.yml @@ -0,0 +1,72 @@ +# Record a deploy in mindsdb/argocd-state, where Argo CD reads what each +# long-lived environment runs: one file per service, /.yaml, and +# this sets its `tag` and `revision`. Argo CD polls that repository and syncs +# the service; nothing here touches a cluster. +# +# `revision` is the commit Argo CD renders the chart from, so pass the PR's +# head sha, which stays fetchable, and the matching `-head-` tag +# build-push-ecr pushes. $GITHUB_SHA on a pull_request run is the synthetic +# merge commit, which GitHub drops once the PR closes. +# +# `token` is an installation token for the mindsdb-release-train App, which +# argocd-state's ruleset lets push to main; mint it in the calling job with +# actions/create-github-app-token (vars.RELEASE_APP_CLIENT_ID and +# secrets.RELEASE_APP_PRIVATE_KEY). The push is retried when another deploy +# lands first. The file has to exist: adding a service is a reviewed change +# in argocd-state. scripts/write_env_state.py does the work; its tests are in +# tests/test_write_env_state.py. + +name: Write environment state +description: Set a service's tag and revision in mindsdb/argocd-state for Argo CD to deploy + +inputs: + env: + description: "The environment, the directory in argocd-state (dev, staging)" + required: true + service: + description: "The service, the file name in argocd-state without .yaml" + required: true + tag: + description: "The image tag to run" + required: true + revision: + description: "The commit sha of the chart to render, 40 characters" + required: true + token: + description: "An installation token for mindsdb-release-train" + required: true + repository: + description: "The state repository" + required: false + default: mindsdb/argocd-state + branch: + description: "The branch Argo CD reads" + required: false + default: main + +runs: + using: composite + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.0.1 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.branch }} + token: ${{ inputs.token }} + path: argocd-state + persist-credentials: true + + - shell: bash + env: + ENV_NAME: ${{ inputs.env }} + SERVICE: ${{ inputs.service }} + TAG: ${{ inputs.tag }} + REVISION: ${{ inputs.revision }} + BRANCH: ${{ inputs.branch }} + ACTION_PATH: ${{ github.action_path }} + run: | + set -euo pipefail + git -C argocd-state config user.name "mindsdb-release-train[bot]" + git -C argocd-state config user.email "mindsdb-release-train[bot]@users.noreply.github.com" + python3 "$ACTION_PATH/../scripts/write_env_state.py" \ + --repo-dir argocd-state --branch "$BRANCH" \ + --env "$ENV_NAME" --service "$SERVICE" --tag "$TAG" --revision "$REVISION" From bd653ef8dd79ed22f334e2be7f4a49111e23cbf4 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 09:17:49 +1300 Subject: [PATCH 2/8] write-env-state: one sha input; the tag is derived from it Co-Authored-By: Claude Fable 5.1 --- scripts/write_env_state.py | 35 +++++++++++++++++++++-------------- tests/test_write_env_state.py | 31 ++++++++++++++++++------------- write-env-state/action.yml | 26 +++++++++++--------------- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/scripts/write_env_state.py b/scripts/write_env_state.py index 65f414c..bb94d69 100644 --- a/scripts/write_env_state.py +++ b/scripts/write_env_state.py @@ -2,9 +2,10 @@ Argo CD deploys each long-lived environment from one file per service in that repository, ``/.yaml`` (its README has the keys). After a build, -CI sets two of them, ``tag`` and ``revision``, and Argo CD does the rest. Every -repository's deploy job does the same thing, so it is written once here and -the ``write-env-state`` action calls it. +CI sets two of them from the PR's head commit: ``revision`` is that sha and +``tag`` is the ``-head-`` tag build-push-ecr pushes +for it, so one value names both. Every repository's deploy job does the same +thing, so it is written once here and the ``write-env-state`` action calls it. The file is edited line by line, not parsed and dumped, so the keys set when the service was added keep their order and comments. It has to exist already: @@ -15,8 +16,7 @@ the branch as it is on the remote, applies the edit again and pushes. A push the remote refuses because the branch moved is the only thing retried. - write_env_state.py --repo-dir --env dev --service scratchpad-controller \ - --tag development-head- --revision + write_env_state.py --repo-dir --env dev --service scratchpad-controller --sha """ from __future__ import annotations @@ -29,23 +29,30 @@ # The same shapes argocd-state's validate.py accepts. LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") -TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$") COMMIT = re.compile(r"^[0-9a-f]{40}$") +# build-push-ecr's build-for-environment for each environment's namespace. +BUILD_ENVIRONMENT = {"dev": "development", "staging": "staging", "prod": "production"} class StateError(Exception): """A reason the deploy cannot be recorded, in words for the job log.""" -def check(env: str, service: str, tag: str, revision: str) -> None: +def check(env: str, service: str, sha: str) -> None: for name, value, pattern, shape in ( ("env", env, LABEL, "a namespace name"), ("service", service, LABEL, "a service name"), - ("tag", tag, TAG, "an image tag"), - ("revision", revision, COMMIT, "a 40-character commit sha"), + ("sha", sha, COMMIT, "a 40-character commit sha"), ): if not pattern.match(value): raise StateError(f"{name} {value!r} is not {shape}") + if env not in BUILD_ENVIRONMENT: + raise StateError(f"env {env!r} has no build environment; known: {', '.join(BUILD_ENVIRONMENT)}") + + +def image_tag(env: str, sha: str) -> str: + """The tag build-push-ecr gave this commit's image on its pull_request build.""" + return f"{BUILD_ENVIRONMENT[env]}-head-{sha}" def updated_text(text: str, tag: str, revision: str) -> str: @@ -69,9 +76,10 @@ def git(repo_dir: Path, *args: str, check: bool = True) -> subprocess.CompletedP return subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True, check=check) -def record(repo_dir: Path, branch: str, env: str, service: str, tag: str, revision: str, attempts: int = 5) -> str: +def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, attempts: int = 5) -> str: """Commit and push the edit; the message says what happened. Raises StateError when it can't.""" - check(env, service, tag, revision) + check(env, service, sha) + tag, revision = image_tag(env, sha), sha path = Path(env) / f"{service}.yaml" for attempt in range(1, attempts + 1): git(repo_dir, "fetch", "--quiet", "origin", branch) @@ -100,12 +108,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--branch", default="main") parser.add_argument("--env", required=True) parser.add_argument("--service", required=True) - parser.add_argument("--tag", required=True) - parser.add_argument("--revision", required=True) + parser.add_argument("--sha", required=True, help="the PR's head commit: the chart revision, and the image tag's suffix") parser.add_argument("--attempts", type=int, default=5) args = parser.parse_args(argv) try: - print(record(args.repo_dir, args.branch, args.env, args.service, args.tag, args.revision, args.attempts)) + print(record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.attempts)) except StateError as error: print(f"::error::{error}") return 1 diff --git a/tests/test_write_env_state.py b/tests/test_write_env_state.py index 71f93a9..484e017 100644 --- a/tests/test_write_env_state.py +++ b/tests/test_write_env_state.py @@ -66,20 +66,25 @@ def test_a_file_without_both_lines_is_refused(self): class TestCheck: @pytest.mark.parametrize("field, value", [ - ("env", "Dev"), ("service", "my_service"), ("tag", "-x"), ("tag", "a b"), - ("revision", "main"), ("revision", SHA[:7]), ("revision", SHA.upper()), + ("env", "Dev"), ("env", "qa"), ("service", "my_service"), + ("sha", "main"), ("sha", SHA[:7]), ("sha", SHA.upper()), ]) def test_each_value_has_a_shape(self, field, value): - values = {"env": "dev", "service": "scratchpad-controller", "tag": "development-head-" + SHA, "revision": SHA} + values = {"env": "dev", "service": "scratchpad-controller", "sha": SHA} values[field] = value with pytest.raises(write_env_state.StateError, match=f"{field} "): write_env_state.check(**values) + def test_the_tag_is_the_head_tag_build_push_ecr_pushed(self): + assert write_env_state.image_tag("dev", SHA) == f"development-head-{SHA}" + assert write_env_state.image_tag("staging", SHA) == f"staging-head-{SHA}" + assert write_env_state.image_tag("prod", SHA) == f"production-head-{SHA}" + class TestRecord: def test_commits_and_pushes_the_edit(self, remote): bare, _, clone = remote - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) assert message.startswith("dev/scratchpad-controller.yaml: tag development-head-") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert git(bare, "log", "-1", "--format=%s", "main") == f"dev/scratchpad-controller: development-head-{SHA}\n" @@ -87,14 +92,14 @@ def test_commits_and_pushes_the_edit(self, remote): def test_nothing_to_do_pushes_nothing(self, remote): bare, _, clone = remote before = git(bare, "rev-parse", "main") - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{OTHER_SHA}", OTHER_SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA) assert message.endswith(f"already records development-head-{OTHER_SHA} at {OTHER_SHA}") assert git(bare, "rev-parse", "main") == before def test_a_missing_file_is_refused_not_created(self, remote): bare, _, clone = remote with pytest.raises(write_env_state.StateError, match="dev/auth.yaml does not exist in argocd-state; adding a service is a reviewed change"): - write_env_state.record(clone, "main", "dev", "auth", "x", SHA) + write_env_state.record(clone, "main", "dev", "auth", SHA) assert "dev/auth.yaml" not in git(bare, "ls-tree", "-r", "--name-only", "main") def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): @@ -104,7 +109,7 @@ def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): git(seed, "add", "-A") git(seed, "commit", "--quiet", "-m", "dev/auth: x") git(seed, "push", "--quiet", "origin", "main") - write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) @@ -125,7 +130,7 @@ def racing_git(repo_dir, *args, check=True): return real_git(repo_dir, *args, check=check) monkeypatch.setattr(write_env_state, "git", racing_git) - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) assert message.endswith("(attempt 2)") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") @@ -146,23 +151,23 @@ def always_racing_git(repo_dir, *args, check=True): monkeypatch.setattr(write_env_state, "git", always_racing_git) with pytest.raises(write_env_state.StateError, match="gave up after 2 attempts"): - write_env_state.record(clone, "main", "dev", "scratchpad-controller", f"development-head-{SHA}", SHA, attempts=2) + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, attempts=2) assert len(pushes) == 2 class TestMain: def test_a_refusal_is_an_error_annotation_and_exit_1(self, remote, capsys): _, _, clone = remote - rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--tag", "x", "--revision", "main"]) + rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--sha", "main"]) assert rc == 1 - assert capsys.readouterr().out.startswith("::error::revision 'main' is not a 40-character commit sha") + assert capsys.readouterr().out.startswith("::error::sha 'main' is not a 40-character commit sha") class TestAction: def test_the_action_calls_the_script_with_every_input(self): import yaml action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) - assert set(action["inputs"]) == {"env", "service", "tag", "revision", "token", "repository", "branch"} + assert set(action["inputs"]) == {"env", "service", "sha", "token", "repository", "branch"} run = action["runs"]["steps"][-1]["run"] - for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--tag "$TAG"', '--revision "$REVISION"'): + for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"'): assert flag in run diff --git a/write-env-state/action.yml b/write-env-state/action.yml index 44f15d1..2917107 100644 --- a/write-env-state/action.yml +++ b/write-env-state/action.yml @@ -1,12 +1,12 @@ # Record a deploy in mindsdb/argocd-state, where Argo CD reads what each # long-lived environment runs: one file per service, /.yaml, and -# this sets its `tag` and `revision`. Argo CD polls that repository and syncs -# the service; nothing here touches a cluster. +# this sets its `tag` and `revision` from one commit sha. Argo CD polls that +# repository and syncs the service; nothing here touches a cluster. # -# `revision` is the commit Argo CD renders the chart from, so pass the PR's -# head sha, which stays fetchable, and the matching `-head-` tag -# build-push-ecr pushes. $GITHUB_SHA on a pull_request run is the synthetic -# merge commit, which GitHub drops once the PR closes. +# `sha` is the PR's head commit: Argo CD renders the chart from it, and the +# image tag is the `-head-` one build-push-ecr pushes +# for it on every pull_request build. Not $GITHUB_SHA: on a pull_request run +# that is the synthetic merge commit, which GitHub drops once the PR closes. # # `token` is an installation token for the mindsdb-release-train App, which # argocd-state's ruleset lets push to main; mint it in the calling job with @@ -17,7 +17,7 @@ # tests/test_write_env_state.py. name: Write environment state -description: Set a service's tag and revision in mindsdb/argocd-state for Argo CD to deploy +description: Record a service's commit in mindsdb/argocd-state for Argo CD to deploy inputs: env: @@ -26,11 +26,8 @@ inputs: service: description: "The service, the file name in argocd-state without .yaml" required: true - tag: - description: "The image tag to run" - required: true - revision: - description: "The commit sha of the chart to render, 40 characters" + sha: + description: "The PR's head commit, 40 characters: the chart revision, and the image tag's suffix" required: true token: description: "An installation token for mindsdb-release-train" @@ -59,8 +56,7 @@ runs: env: ENV_NAME: ${{ inputs.env }} SERVICE: ${{ inputs.service }} - TAG: ${{ inputs.tag }} - REVISION: ${{ inputs.revision }} + SHA: ${{ inputs.sha }} BRANCH: ${{ inputs.branch }} ACTION_PATH: ${{ github.action_path }} run: | @@ -69,4 +65,4 @@ runs: git -C argocd-state config user.email "mindsdb-release-train[bot]@users.noreply.github.com" python3 "$ACTION_PATH/../scripts/write_env_state.py" \ --repo-dir argocd-state --branch "$BRANCH" \ - --env "$ENV_NAME" --service "$SERVICE" --tag "$TAG" --revision "$REVISION" + --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" From 002e91a43b2869f42bf656d4ece2258ca7f9e5b0 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 09:21:19 +1300 Subject: [PATCH 3/8] write-env-state: the caller passes the tag Co-Authored-By: Claude Fable 5.1 --- scripts/write_env_state.py | 35 +++++++++++++++-------------------- tests/test_write_env_state.py | 27 +++++++++++---------------- write-env-state/action.yml | 22 +++++++++++++--------- 3 files changed, 39 insertions(+), 45 deletions(-) diff --git a/scripts/write_env_state.py b/scripts/write_env_state.py index bb94d69..9a91b59 100644 --- a/scripts/write_env_state.py +++ b/scripts/write_env_state.py @@ -2,10 +2,10 @@ Argo CD deploys each long-lived environment from one file per service in that repository, ``/.yaml`` (its README has the keys). After a build, -CI sets two of them from the PR's head commit: ``revision`` is that sha and -``tag`` is the ``-head-`` tag build-push-ecr pushes -for it, so one value names both. Every repository's deploy job does the same -thing, so it is written once here and the ``write-env-state`` action calls it. +CI sets two of them: ``revision``, the commit Argo CD renders the chart from, +and ``tag``, the image build-push-ecr pushed for it. Every repository's deploy +job does the same thing, so it is written once here and the +``write-env-state`` action calls it. The file is edited line by line, not parsed and dumped, so the keys set when the service was added keep their order and comments. It has to exist already: @@ -16,7 +16,8 @@ the branch as it is on the remote, applies the edit again and pushes. A push the remote refuses because the branch moved is the only thing retried. - write_env_state.py --repo-dir --env dev --service scratchpad-controller --sha + write_env_state.py --repo-dir --env dev --service scratchpad-controller \ + --sha --tag development-head- """ from __future__ import annotations @@ -29,30 +30,23 @@ # The same shapes argocd-state's validate.py accepts. LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") +TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$") COMMIT = re.compile(r"^[0-9a-f]{40}$") -# build-push-ecr's build-for-environment for each environment's namespace. -BUILD_ENVIRONMENT = {"dev": "development", "staging": "staging", "prod": "production"} class StateError(Exception): """A reason the deploy cannot be recorded, in words for the job log.""" -def check(env: str, service: str, sha: str) -> None: +def check(env: str, service: str, sha: str, tag: str) -> None: for name, value, pattern, shape in ( ("env", env, LABEL, "a namespace name"), ("service", service, LABEL, "a service name"), ("sha", sha, COMMIT, "a 40-character commit sha"), + ("tag", tag, TAG, "an image tag"), ): if not pattern.match(value): raise StateError(f"{name} {value!r} is not {shape}") - if env not in BUILD_ENVIRONMENT: - raise StateError(f"env {env!r} has no build environment; known: {', '.join(BUILD_ENVIRONMENT)}") - - -def image_tag(env: str, sha: str) -> str: - """The tag build-push-ecr gave this commit's image on its pull_request build.""" - return f"{BUILD_ENVIRONMENT[env]}-head-{sha}" def updated_text(text: str, tag: str, revision: str) -> str: @@ -76,10 +70,10 @@ def git(repo_dir: Path, *args: str, check: bool = True) -> subprocess.CompletedP return subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True, check=check) -def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, attempts: int = 5) -> str: +def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, tag: str, attempts: int = 5) -> str: """Commit and push the edit; the message says what happened. Raises StateError when it can't.""" - check(env, service, sha) - tag, revision = image_tag(env, sha), sha + check(env, service, sha, tag) + revision = sha path = Path(env) / f"{service}.yaml" for attempt in range(1, attempts + 1): git(repo_dir, "fetch", "--quiet", "origin", branch) @@ -108,11 +102,12 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--branch", default="main") parser.add_argument("--env", required=True) parser.add_argument("--service", required=True) - parser.add_argument("--sha", required=True, help="the PR's head commit: the chart revision, and the image tag's suffix") + parser.add_argument("--sha", required=True, help="the commit Argo CD renders the chart from") + parser.add_argument("--tag", required=True, help="the image tag build-push-ecr pushed for it") parser.add_argument("--attempts", type=int, default=5) args = parser.parse_args(argv) try: - print(record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.attempts)) + print(record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.tag, args.attempts)) except StateError as error: print(f"::error::{error}") return 1 diff --git a/tests/test_write_env_state.py b/tests/test_write_env_state.py index 484e017..99f1a56 100644 --- a/tests/test_write_env_state.py +++ b/tests/test_write_env_state.py @@ -66,25 +66,20 @@ def test_a_file_without_both_lines_is_refused(self): class TestCheck: @pytest.mark.parametrize("field, value", [ - ("env", "Dev"), ("env", "qa"), ("service", "my_service"), + ("env", "Dev"), ("service", "my_service"), ("tag", "-x"), ("tag", "a b"), ("sha", "main"), ("sha", SHA[:7]), ("sha", SHA.upper()), ]) def test_each_value_has_a_shape(self, field, value): - values = {"env": "dev", "service": "scratchpad-controller", "sha": SHA} + values = {"env": "dev", "service": "scratchpad-controller", "sha": SHA, "tag": f"development-head-{SHA}"} values[field] = value with pytest.raises(write_env_state.StateError, match=f"{field} "): write_env_state.check(**values) - def test_the_tag_is_the_head_tag_build_push_ecr_pushed(self): - assert write_env_state.image_tag("dev", SHA) == f"development-head-{SHA}" - assert write_env_state.image_tag("staging", SHA) == f"staging-head-{SHA}" - assert write_env_state.image_tag("prod", SHA) == f"production-head-{SHA}" - class TestRecord: def test_commits_and_pushes_the_edit(self, remote): bare, _, clone = remote - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") assert message.startswith("dev/scratchpad-controller.yaml: tag development-head-") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert git(bare, "log", "-1", "--format=%s", "main") == f"dev/scratchpad-controller: development-head-{SHA}\n" @@ -92,14 +87,14 @@ def test_commits_and_pushes_the_edit(self, remote): def test_nothing_to_do_pushes_nothing(self, remote): bare, _, clone = remote before = git(bare, "rev-parse", "main") - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA, f"development-head-{OTHER_SHA}") assert message.endswith(f"already records development-head-{OTHER_SHA} at {OTHER_SHA}") assert git(bare, "rev-parse", "main") == before def test_a_missing_file_is_refused_not_created(self, remote): bare, _, clone = remote with pytest.raises(write_env_state.StateError, match="dev/auth.yaml does not exist in argocd-state; adding a service is a reviewed change"): - write_env_state.record(clone, "main", "dev", "auth", SHA) + write_env_state.record(clone, "main", "dev", "auth", SHA, "x") assert "dev/auth.yaml" not in git(bare, "ls-tree", "-r", "--name-only", "main") def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): @@ -109,7 +104,7 @@ def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): git(seed, "add", "-A") git(seed, "commit", "--quiet", "-m", "dev/auth: x") git(seed, "push", "--quiet", "origin", "main") - write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) @@ -130,7 +125,7 @@ def racing_git(repo_dir, *args, check=True): return real_git(repo_dir, *args, check=check) monkeypatch.setattr(write_env_state, "git", racing_git) - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA) + message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") assert message.endswith("(attempt 2)") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") @@ -151,14 +146,14 @@ def always_racing_git(repo_dir, *args, check=True): monkeypatch.setattr(write_env_state, "git", always_racing_git) with pytest.raises(write_env_state.StateError, match="gave up after 2 attempts"): - write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, attempts=2) + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}", attempts=2) assert len(pushes) == 2 class TestMain: def test_a_refusal_is_an_error_annotation_and_exit_1(self, remote, capsys): _, _, clone = remote - rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--sha", "main"]) + rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--sha", "main", "--tag", "x"]) assert rc == 1 assert capsys.readouterr().out.startswith("::error::sha 'main' is not a 40-character commit sha") @@ -167,7 +162,7 @@ class TestAction: def test_the_action_calls_the_script_with_every_input(self): import yaml action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) - assert set(action["inputs"]) == {"env", "service", "sha", "token", "repository", "branch"} + assert set(action["inputs"]) == {"env", "service", "sha", "tag", "token", "repository", "branch"} run = action["runs"]["steps"][-1]["run"] - for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"'): + for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"', '--tag "$TAG"'): assert flag in run diff --git a/write-env-state/action.yml b/write-env-state/action.yml index 2917107..d1826c9 100644 --- a/write-env-state/action.yml +++ b/write-env-state/action.yml @@ -1,12 +1,12 @@ # Record a deploy in mindsdb/argocd-state, where Argo CD reads what each # long-lived environment runs: one file per service, /.yaml, and -# this sets its `tag` and `revision` from one commit sha. Argo CD polls that -# repository and syncs the service; nothing here touches a cluster. +# this sets its `revision` and `tag`. Argo CD polls that repository and syncs +# the service; nothing here touches a cluster. # -# `sha` is the PR's head commit: Argo CD renders the chart from it, and the -# image tag is the `-head-` one build-push-ecr pushes -# for it on every pull_request build. Not $GITHUB_SHA: on a pull_request run -# that is the synthetic merge commit, which GitHub drops once the PR closes. +# `sha` is the commit Argo CD renders the chart from and `tag` the image +# build-push-ecr pushed for it. On a pull_request run pass the PR's head sha +# and its `-head-` tag, not $GITHUB_SHA: that is the +# synthetic merge commit, which GitHub drops once the PR closes. # # `token` is an installation token for the mindsdb-release-train App, which # argocd-state's ruleset lets push to main; mint it in the calling job with @@ -17,7 +17,7 @@ # tests/test_write_env_state.py. name: Write environment state -description: Record a service's commit in mindsdb/argocd-state for Argo CD to deploy +description: Record a service's commit and image tag in mindsdb/argocd-state for Argo CD to deploy inputs: env: @@ -27,7 +27,10 @@ inputs: description: "The service, the file name in argocd-state without .yaml" required: true sha: - description: "The PR's head commit, 40 characters: the chart revision, and the image tag's suffix" + description: "The commit Argo CD renders the chart from, 40 characters" + required: true + tag: + description: "The image tag build-push-ecr pushed for that commit" required: true token: description: "An installation token for mindsdb-release-train" @@ -57,6 +60,7 @@ runs: ENV_NAME: ${{ inputs.env }} SERVICE: ${{ inputs.service }} SHA: ${{ inputs.sha }} + TAG: ${{ inputs.tag }} BRANCH: ${{ inputs.branch }} ACTION_PATH: ${{ github.action_path }} run: | @@ -65,4 +69,4 @@ runs: git -C argocd-state config user.email "mindsdb-release-train[bot]@users.noreply.github.com" python3 "$ACTION_PATH/../scripts/write_env_state.py" \ --repo-dir argocd-state --branch "$BRANCH" \ - --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" + --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" --tag "$TAG" From 96f02bb6cbae5fa4de2b5026faf35084d3b0542a Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 09:27:46 +1300 Subject: [PATCH 4/8] build-push-ecr: remote-builder input for GitHub-hosted runners Co-Authored-By: Claude Fable 5.1 --- build-push-ecr/action.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index e02232f..2d02593 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -28,6 +28,10 @@ inputs: image-ref: description: "The version number or sha used in creating image tag" required: false + remote-builder: + description: "Build on the in-cluster BuildKit agent, which only a runner inside newdev can reach. Set 'false' on a GitHub-hosted runner: the build then runs on the runner's own builder with the GitHub Actions cache." + required: false + default: "true" runs: @@ -56,6 +60,7 @@ runs: BRANCH_NAME=${{env.ENV_NAME}} IMAGE_TAG=$ENVIRONMENT-$IMAGE_REF PR_HEAD_SHA="${{ github.event.pull_request.head.sha }}" + REMOTE_BUILDER="${{ inputs.remote-builder }}" # On pull_request events, also tag by the immutable PR head SHA (see # header comment). HEAD_TAG is expanded unquoted on the build line — @@ -70,10 +75,17 @@ runs: aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || \ true # Just let this fail if the repo already exists - docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + CACHE_ARGS="" + if [ "$REMOTE_BUILDER" = "false" ]; then + # setup-buildx-action gave this runner a builder of its own; the + # GitHub Actions cache stands in for the agent's layer cache. + CACHE_ARGS="--cache-from type=gha --cache-to type=gha,mode=max" + else + docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + fi cd $SRC_PATH - BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG" + BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG $CACHE_ARGS" # Finally, build our runner container docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS $HEAD_TAG -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:$ENVIRONMENT -t $REPO_IMAGE:latest --push . From e10dbe714698d42f7d343a9b4d693b679746d3be Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 14:32:31 +1300 Subject: [PATCH 5/8] write-env-state: set the argocd/ status to pending before the push Argo CD's notifier only ever sets that status to success or failure, so a sync that never starts left the commit with no status at all. Set pending first and a stuck sync shows as a pending that never clears; a failed push sets it to error with the run as its link. Co-Authored-By: Claude Fable 5.1 --- write-env-state/action.yml | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/write-env-state/action.yml b/write-env-state/action.yml index d1826c9..dbe4262 100644 --- a/write-env-state/action.yml +++ b/write-env-state/action.yml @@ -15,6 +15,11 @@ # lands first. The file has to exist: adding a service is a reviewed change # in argocd-state. scripts/write_env_state.py does the work; its tests are in # tests/test_write_env_state.py. +# +# Before the push, commit status `argocd/` is set to pending on `sha` +# in the calling repository: Argo CD's notifier sets the same status to +# success or failure once it has synced, so a sync that never starts shows +# as a pending that never clears. The calling job needs `statuses: write`. name: Write environment state description: Record a service's commit and image tag in mindsdb/argocd-state for Argo CD to deploy @@ -43,6 +48,14 @@ inputs: description: "The branch Argo CD reads" required: false default: main + status-token: + description: "A token that may write commit statuses on the calling repository" + required: false + default: ${{ github.token }} + argocd-url: + description: "The Argo CD that deploys this environment, for the status link" + required: false + default: https://argo.dev.mindsdb.com runs: using: composite @@ -55,6 +68,22 @@ runs: path: argocd-state persist-credentials: true + - name: Mark the commit as waiting for Argo CD + shell: bash + env: + GH_TOKEN: ${{ inputs.status-token }} + REPOSITORY: ${{ github.repository }} + ENV_NAME: ${{ inputs.env }} + SERVICE: ${{ inputs.service }} + SHA: ${{ inputs.sha }} + ARGOCD_URL: ${{ inputs.argocd-url }} + run: | + set -euo pipefail + gh api "repos/$REPOSITORY/statuses/$SHA" \ + -f state=pending -f context="argocd/$ENV_NAME" \ + -f description="Waiting for Argo CD to sync" \ + -f target_url="$ARGOCD_URL/applications/$ENV_NAME-$SERVICE?operation=true" > /dev/null + - shell: bash env: ENV_NAME: ${{ inputs.env }} @@ -70,3 +99,20 @@ runs: python3 "$ACTION_PATH/../scripts/write_env_state.py" \ --repo-dir argocd-state --branch "$BRANCH" \ --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" --tag "$TAG" + + - name: Mark the commit as not recorded + # The pending above would otherwise outlive a failed push. + if: failure() + shell: bash + env: + GH_TOKEN: ${{ inputs.status-token }} + REPOSITORY: ${{ github.repository }} + ENV_NAME: ${{ inputs.env }} + SHA: ${{ inputs.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh api "repos/$REPOSITORY/statuses/$SHA" \ + -f state=error -f context="argocd/$ENV_NAME" \ + -f description="The deploy was not recorded in argocd-state" \ + -f target_url="$RUN_URL" > /dev/null From 260d84e12907bb3bdf25c62517a0289e5c4bd951 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 16:47:20 +1300 Subject: [PATCH 6/8] build-push-ecr: drop the gha cache flags; read remote-builder from env buildx takes the Actions cache token and URL from variables a run: step never has, so --cache-to type=gha was dropped without a word: the hosted build on this branch logged no cache import or export. remote-builder "false" now only skips the remote builder; "true", the default, is the same command as before. Co-Authored-By: Claude Fable 5.1 --- build-push-ecr/action.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index 2d02593..369393f 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -29,7 +29,7 @@ inputs: description: "The version number or sha used in creating image tag" required: false remote-builder: - description: "Build on the in-cluster BuildKit agent, which only a runner inside newdev can reach. Set 'false' on a GitHub-hosted runner: the build then runs on the runner's own builder with the GitHub Actions cache." + description: "Build on the in-cluster BuildKit agent, which only a runner inside newdev can reach. Set 'false' on a GitHub-hosted runner: the build then runs on the builder setup-buildx-action made on the runner." required: false default: "true" @@ -46,6 +46,8 @@ runs: uses: aws-actions/amazon-ecr-login@v2 - shell: bash id: build + env: + REMOTE_BUILDER: ${{ inputs.remote-builder }} run: | # Env var parsing @@ -60,7 +62,6 @@ runs: BRANCH_NAME=${{env.ENV_NAME}} IMAGE_TAG=$ENVIRONMENT-$IMAGE_REF PR_HEAD_SHA="${{ github.event.pull_request.head.sha }}" - REMOTE_BUILDER="${{ inputs.remote-builder }}" # On pull_request events, also tag by the immutable PR head SHA (see # header comment). HEAD_TAG is expanded unquoted on the build line — @@ -75,17 +76,12 @@ runs: aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || \ true # Just let this fail if the repo already exists - CACHE_ARGS="" - if [ "$REMOTE_BUILDER" = "false" ]; then - # setup-buildx-action gave this runner a builder of its own; the - # GitHub Actions cache stands in for the agent's layer cache. - CACHE_ARGS="--cache-from type=gha --cache-to type=gha,mode=max" - else + if [ "$REMOTE_BUILDER" != "false" ]; then docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) fi cd $SRC_PATH - BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG $CACHE_ARGS" + BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG" # Finally, build our runner container docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS $HEAD_TAG -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:$ENVIRONMENT -t $REPO_IMAGE:latest --push . From 936ad10e2420845a3c9da941024204376d36c9f2 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 16:47:20 +1300 Subject: [PATCH 7/8] write_env_state: retry only a race, keep git's reasons, keep comments "rejected" also matched a push a ruleset or hook refused, so that was tried five times and reported as the branch moving. Only git's race messages are retried now; a refusal raises with git's own text after one attempt, and so does any other failed git command. A comment after the tag or revision value stays. record() says whether it pushed, and main writes pushed=true|false to --github-output for the action. Co-Authored-By: Claude Fable 5.1 --- scripts/write_env_state.py | 41 +++++++++++++++------ tests/test_write_env_state.py | 68 +++++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/scripts/write_env_state.py b/scripts/write_env_state.py index 9a91b59..ea5cfb1 100644 --- a/scripts/write_env_state.py +++ b/scripts/write_env_state.py @@ -49,16 +49,20 @@ def check(env: str, service: str, sha: str, tag: str) -> None: raise StateError(f"{name} {value!r} is not {shape}") +LINE = re.compile(r"^(?Ptag|revision):[^#\r\n]*?(?P[ \t]+#[^\r\n]*)?(?P\r?\n?)$") + + def updated_text(text: str, tag: str, revision: str) -> str: - """The state file with its tag and revision lines replaced, everything else as it was.""" + """The state file with its tag and revision values replaced, everything else as it was: a comment after the value stays.""" out = [] seen = set() for line in text.splitlines(keepends=True): - key = line.split(":", 1)[0] if ":" in line else None - if key in ("tag", "revision") and not line.startswith((" ", "#")): + match = LINE.match(line) + if match: + key = match["key"] seen.add(key) value = tag if key == "tag" else revision - line = f"{key}: {value}\n" + line = f"{key}: {value}{match['comment'] or ''}{match['eol']}" out.append(line) missing = {"tag", "revision"} - seen if missing: @@ -67,11 +71,21 @@ def updated_text(text: str, tag: str, revision: str) -> str: def git(repo_dir: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True, check=check) + """Run git in the clone; with check, a failure is a StateError carrying git's own message.""" + result = subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True) + if check and result.returncode != 0: + raise StateError(f"git {args[0]} failed: {result.stderr.strip()}") + return result + + +# What git says when the push lost a race, client side (the branch moved since +# the fetch) or server side (two pushes reached the server at once). Anything +# else, a ruleset or a hook refusing the push, is not retried. +RACE_MARKERS = ("(fetch first)", "(non-fast-forward)", "(incorrect old value provided)", "cannot lock ref") -def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, tag: str, attempts: int = 5) -> str: - """Commit and push the edit; the message says what happened. Raises StateError when it can't.""" +def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, tag: str, attempts: int = 5) -> tuple[bool, str]: + """Commit and push the edit. Returns whether a commit was pushed and a message saying what happened; raises StateError when it can't.""" check(env, service, sha, tag) revision = sha path = Path(env) / f"{service}.yaml" @@ -84,14 +98,14 @@ def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, tag: s text = file.read_text(encoding="utf-8") new = updated_text(text, tag, revision) if new == text: - return f"{path} already records {tag} at {revision}" + return False, f"{path} already records {tag} at {revision}" file.write_text(new, encoding="utf-8") git(repo_dir, "add", str(path)) git(repo_dir, "commit", "--quiet", "-m", f"{env}/{service}: {tag}") push = git(repo_dir, "push", "--quiet", "origin", f"HEAD:{branch}", check=False) if push.returncode == 0: - return f"{path}: tag {tag}, revision {revision} (attempt {attempt})" - if "rejected" not in push.stderr and "fetch first" not in push.stderr and "non-fast-forward" not in push.stderr: + return True, f"{path}: tag {tag}, revision {revision} (attempt {attempt})" + if not any(marker in push.stderr for marker in RACE_MARKERS): raise StateError(f"push to {branch} failed: {push.stderr.strip()}") raise StateError(f"{branch} kept moving; gave up after {attempts} attempts") @@ -105,12 +119,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--sha", required=True, help="the commit Argo CD renders the chart from") parser.add_argument("--tag", required=True, help="the image tag build-push-ecr pushed for it") parser.add_argument("--attempts", type=int, default=5) + parser.add_argument("--github-output", type=Path, help="a file to append pushed=true|false to, as $GITHUB_OUTPUT") args = parser.parse_args(argv) try: - print(record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.tag, args.attempts)) + pushed, message = record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.tag, args.attempts) except StateError as error: print(f"::error::{error}") return 1 + print(message) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as out: + out.write(f"pushed={'true' if pushed else 'false'}\n") return 0 diff --git a/tests/test_write_env_state.py b/tests/test_write_env_state.py index 99f1a56..59e464c 100644 --- a/tests/test_write_env_state.py +++ b/tests/test_write_env_state.py @@ -57,6 +57,13 @@ def test_replaces_only_the_two_lines(self): assert out == STATE.replace(OTHER_SHA, SHA) assert "# the image CI last built\n" in out + def test_a_comment_after_the_value_stays(self): + # argocd-state's README template puts one on each of the two lines. + text = STATE.replace(f"tag: development-head-{OTHER_SHA}\n", f"tag: development-head-{OTHER_SHA} # the image tag, written by CI\n") + out = write_env_state.updated_text(text, f"development-head-{SHA}", SHA) + assert f"tag: development-head-{SHA} # the image tag, written by CI\n" in out + assert f"revision: {SHA}\n" in out + def test_a_file_without_both_lines_is_refused(self): with pytest.raises(write_env_state.StateError, match="no revision line"): write_env_state.updated_text("tag: x\n", "y", SHA) @@ -76,10 +83,18 @@ def test_each_value_has_a_shape(self, field, value): write_env_state.check(**values) +class TestGit: + def test_a_failed_command_is_a_state_error_with_gits_reason(self, remote): + _, _, clone = remote + with pytest.raises(write_env_state.StateError, match="git fetch failed: fatal: 'nowhere' does not appear to be a git repository"): + write_env_state.git(clone, "fetch", "nowhere") + + class TestRecord: def test_commits_and_pushes_the_edit(self, remote): bare, _, clone = remote - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert pushed assert message.startswith("dev/scratchpad-controller.yaml: tag development-head-") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert git(bare, "log", "-1", "--format=%s", "main") == f"dev/scratchpad-controller: development-head-{SHA}\n" @@ -87,7 +102,8 @@ def test_commits_and_pushes_the_edit(self, remote): def test_nothing_to_do_pushes_nothing(self, remote): bare, _, clone = remote before = git(bare, "rev-parse", "main") - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA, f"development-head-{OTHER_SHA}") + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA, f"development-head-{OTHER_SHA}") + assert not pushed assert message.endswith(f"already records development-head-{OTHER_SHA} at {OTHER_SHA}") assert git(bare, "rev-parse", "main") == before @@ -125,7 +141,8 @@ def racing_git(repo_dir, *args, check=True): return real_git(repo_dir, *args, check=check) monkeypatch.setattr(write_env_state, "git", racing_git) - message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert pushed assert message.endswith("(attempt 2)") assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") @@ -150,7 +167,36 @@ def always_racing_git(repo_dir, *args, check=True): assert len(pushes) == 2 + def test_a_push_the_server_refuses_is_not_retried(self, remote, monkeypatch): + # A ruleset or hook saying no is final; its reason is the error, and the branch is not "moving". + bare, _, clone = remote + hook = bare / "hooks" / "pre-receive" + hook.write_text("#!/bin/sh\necho 'no deploys during the freeze' >&2\nexit 1\n") + hook.chmod(0o755) + real_git = write_env_state.git + pushes = [] + + def counting_git(repo_dir, *args, check=True): + if args[0] == "push": + pushes.append(True) + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", counting_git) + with pytest.raises(write_env_state.StateError, match="push to main failed: .*no deploys during the freeze"): + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert len(pushes) == 1 + + class TestMain: + def test_says_whether_a_commit_was_pushed(self, remote, tmp_path, capsys): + _, _, clone = remote + output = tmp_path / "github-output" + base = ["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--github-output", str(output)] + assert write_env_state.main(base + ["--sha", SHA, "--tag", f"development-head-{SHA}"]) == 0 + assert write_env_state.main(base + ["--sha", SHA, "--tag", f"development-head-{SHA}"]) == 0 + assert output.read_text() == "pushed=true\npushed=false\n" + assert "(attempt 1)" in capsys.readouterr().out + def test_a_refusal_is_an_error_annotation_and_exit_1(self, remote, capsys): _, _, clone = remote rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--sha", "main", "--tag", "x"]) @@ -162,7 +208,17 @@ class TestAction: def test_the_action_calls_the_script_with_every_input(self): import yaml action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) - assert set(action["inputs"]) == {"env", "service", "sha", "tag", "token", "repository", "branch"} - run = action["runs"]["steps"][-1]["run"] - for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"', '--tag "$TAG"'): + assert set(action["inputs"]) == {"env", "service", "sha", "tag", "token", "repository", "branch", "status-token", "argocd-url"} + steps = {step.get("id") or step.get("name"): step for step in action["runs"]["steps"]} + run = steps["write"]["run"] + for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"', '--tag "$TAG"', '--github-output "$GITHUB_OUTPUT"'): assert flag in run + + def test_the_pending_status_follows_a_pushed_commit_only(self): + import yaml + action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) + steps = action["runs"]["steps"] + pending = next(step for step in steps if step.get("name") == "Mark the commit as waiting for Argo CD") + assert steps.index(pending) > steps.index(next(step for step in steps if step.get("id") == "write")) + assert pending["if"] == "steps.write.outputs.pushed == 'true'" + assert "state=pending" in pending["run"] and 'context="argocd/$ENV_NAME"' in pending["run"] From 8dadb04c212e1d21ce96fa5a32749e686d4e7eed Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Fri, 13 Nov 2026 16:47:20 +1300 Subject: [PATCH 8/8] write-env-state: set pending only after a commit was pushed A rerun whose file already matched set the pending before the script returned without a commit, and nothing ever cleared it. The pending step now follows the script and runs on its pushed output; the error step went with the reordering. Checkout pin comment corrected to v5.1.0. Co-Authored-By: Claude Fable 5.1 --- write-env-state/action.yml | 46 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/write-env-state/action.yml b/write-env-state/action.yml index dbe4262..fcfe35c 100644 --- a/write-env-state/action.yml +++ b/write-env-state/action.yml @@ -16,10 +16,12 @@ # in argocd-state. scripts/write_env_state.py does the work; its tests are in # tests/test_write_env_state.py. # -# Before the push, commit status `argocd/` is set to pending on `sha` -# in the calling repository: Argo CD's notifier sets the same status to -# success or failure once it has synced, so a sync that never starts shows -# as a pending that never clears. The calling job needs `statuses: write`. +# When a commit was pushed, commit status `argocd/` is set to pending +# on `sha` in the calling repository: Argo CD's notifier sets the same +# status to success or failure once it has synced, so a sync that never +# starts shows as a pending that never clears. A rerun that finds the file +# already up to date sets nothing, since nothing will sync. The calling job +# needs `statuses: write`. name: Write environment state description: Record a service's commit and image tag in mindsdb/argocd-state for Argo CD to deploy @@ -60,7 +62,7 @@ inputs: runs: using: composite steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.0.1 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: repository: ${{ inputs.repository }} ref: ${{ inputs.branch }} @@ -68,23 +70,8 @@ runs: path: argocd-state persist-credentials: true - - name: Mark the commit as waiting for Argo CD + - id: write shell: bash - env: - GH_TOKEN: ${{ inputs.status-token }} - REPOSITORY: ${{ github.repository }} - ENV_NAME: ${{ inputs.env }} - SERVICE: ${{ inputs.service }} - SHA: ${{ inputs.sha }} - ARGOCD_URL: ${{ inputs.argocd-url }} - run: | - set -euo pipefail - gh api "repos/$REPOSITORY/statuses/$SHA" \ - -f state=pending -f context="argocd/$ENV_NAME" \ - -f description="Waiting for Argo CD to sync" \ - -f target_url="$ARGOCD_URL/applications/$ENV_NAME-$SERVICE?operation=true" > /dev/null - - - shell: bash env: ENV_NAME: ${{ inputs.env }} SERVICE: ${{ inputs.service }} @@ -98,21 +85,22 @@ runs: git -C argocd-state config user.email "mindsdb-release-train[bot]@users.noreply.github.com" python3 "$ACTION_PATH/../scripts/write_env_state.py" \ --repo-dir argocd-state --branch "$BRANCH" \ - --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" --tag "$TAG" + --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" --tag "$TAG" \ + --github-output "$GITHUB_OUTPUT" - - name: Mark the commit as not recorded - # The pending above would otherwise outlive a failed push. - if: failure() + - name: Mark the commit as waiting for Argo CD + if: steps.write.outputs.pushed == 'true' shell: bash env: GH_TOKEN: ${{ inputs.status-token }} REPOSITORY: ${{ github.repository }} ENV_NAME: ${{ inputs.env }} + SERVICE: ${{ inputs.service }} SHA: ${{ inputs.sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ARGOCD_URL: ${{ inputs.argocd-url }} run: | set -euo pipefail gh api "repos/$REPOSITORY/statuses/$SHA" \ - -f state=error -f context="argocd/$ENV_NAME" \ - -f description="The deploy was not recorded in argocd-state" \ - -f target_url="$RUN_URL" > /dev/null + -f state=pending -f context="argocd/$ENV_NAME" \ + -f description="Waiting for Argo CD to sync" \ + -f target_url="$ARGOCD_URL/applications/$ENV_NAME-$SERVICE?operation=true" > /dev/null