diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index e02232f..369393f 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -28,6 +28,10 @@ inputs: image-ref: description: "The version number or sha used in creating image tag" required: false + remote-builder: + description: "Build on the in-cluster BuildKit agent, which only a runner inside newdev can reach. Set 'false' on a GitHub-hosted runner: the build then runs on the builder setup-buildx-action made on the runner." + required: false + default: "true" runs: @@ -42,6 +46,8 @@ runs: uses: aws-actions/amazon-ecr-login@v2 - shell: bash id: build + env: + REMOTE_BUILDER: ${{ inputs.remote-builder }} run: | # Env var parsing @@ -70,7 +76,9 @@ runs: aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || \ true # Just let this fail if the repo already exists - docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + if [ "$REMOTE_BUILDER" != "false" ]; then + docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + fi cd $SRC_PATH BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG" diff --git a/scripts/write_env_state.py b/scripts/write_env_state.py new file mode 100644 index 0000000..ea5cfb1 --- /dev/null +++ b/scripts/write_env_state.py @@ -0,0 +1,137 @@ +"""Record a deploy in mindsdb/argocd-state. + +Argo CD deploys each long-lived environment from one file per service in that +repository, ``/.yaml`` (its README has the keys). After a build, +CI sets two of them: ``revision``, the commit Argo CD renders the chart from, +and ``tag``, the image build-push-ecr pushed for it. Every repository's deploy +job does the same thing, so it is written once here and the +``write-env-state`` action calls it. + +The file is edited line by line, not parsed and dumped, so the keys set when +the service was added keep their order and comments. It has to exist already: +adding a service means choosing its chart path and release name, which is a +reviewed change in argocd-state, not something a deploy creates on the way. + +Two deploys can land at once, so the push is retried: each attempt starts from +the branch as it is on the remote, applies the edit again and pushes. A push +the remote refuses because the branch moved is the only thing retried. + + write_env_state.py --repo-dir --env dev --service scratchpad-controller \ + --sha --tag development-head- +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +# The same shapes argocd-state's validate.py accepts. +LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") +TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +class StateError(Exception): + """A reason the deploy cannot be recorded, in words for the job log.""" + + +def check(env: str, service: str, sha: str, tag: str) -> None: + for name, value, pattern, shape in ( + ("env", env, LABEL, "a namespace name"), + ("service", service, LABEL, "a service name"), + ("sha", sha, COMMIT, "a 40-character commit sha"), + ("tag", tag, TAG, "an image tag"), + ): + if not pattern.match(value): + raise StateError(f"{name} {value!r} is not {shape}") + + +LINE = re.compile(r"^(?Ptag|revision):[^#\r\n]*?(?P[ \t]+#[^\r\n]*)?(?P\r?\n?)$") + + +def updated_text(text: str, tag: str, revision: str) -> str: + """The state file with its tag and revision values replaced, everything else as it was: a comment after the value stays.""" + out = [] + seen = set() + for line in text.splitlines(keepends=True): + match = LINE.match(line) + if match: + key = match["key"] + seen.add(key) + value = tag if key == "tag" else revision + line = f"{key}: {value}{match['comment'] or ''}{match['eol']}" + out.append(line) + missing = {"tag", "revision"} - seen + if missing: + raise StateError(f"the file has no {' or '.join(sorted(missing))} line") + return "".join(out) + + +def git(repo_dir: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess: + """Run git in the clone; with check, a failure is a StateError carrying git's own message.""" + result = subprocess.run(["git", "-C", str(repo_dir), *args], capture_output=True, text=True) + if check and result.returncode != 0: + raise StateError(f"git {args[0]} failed: {result.stderr.strip()}") + return result + + +# What git says when the push lost a race, client side (the branch moved since +# the fetch) or server side (two pushes reached the server at once). Anything +# else, a ruleset or a hook refusing the push, is not retried. +RACE_MARKERS = ("(fetch first)", "(non-fast-forward)", "(incorrect old value provided)", "cannot lock ref") + + +def record(repo_dir: Path, branch: str, env: str, service: str, sha: str, tag: str, attempts: int = 5) -> tuple[bool, str]: + """Commit and push the edit. Returns whether a commit was pushed and a message saying what happened; raises StateError when it can't.""" + check(env, service, sha, tag) + revision = sha + path = Path(env) / f"{service}.yaml" + for attempt in range(1, attempts + 1): + git(repo_dir, "fetch", "--quiet", "origin", branch) + git(repo_dir, "reset", "--quiet", "--hard", f"origin/{branch}") + file = repo_dir / path + if not file.is_file(): + raise StateError(f"{path} does not exist in argocd-state; adding a service is a reviewed change there, not a deploy") + text = file.read_text(encoding="utf-8") + new = updated_text(text, tag, revision) + if new == text: + return False, f"{path} already records {tag} at {revision}" + file.write_text(new, encoding="utf-8") + git(repo_dir, "add", str(path)) + git(repo_dir, "commit", "--quiet", "-m", f"{env}/{service}: {tag}") + push = git(repo_dir, "push", "--quiet", "origin", f"HEAD:{branch}", check=False) + if push.returncode == 0: + return True, f"{path}: tag {tag}, revision {revision} (attempt {attempt})" + if not any(marker in push.stderr for marker in RACE_MARKERS): + raise StateError(f"push to {branch} failed: {push.stderr.strip()}") + raise StateError(f"{branch} kept moving; gave up after {attempts} attempts") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument("--repo-dir", type=Path, required=True, help="a clone of argocd-state with origin set") + parser.add_argument("--branch", default="main") + parser.add_argument("--env", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--sha", required=True, help="the commit Argo CD renders the chart from") + parser.add_argument("--tag", required=True, help="the image tag build-push-ecr pushed for it") + parser.add_argument("--attempts", type=int, default=5) + parser.add_argument("--github-output", type=Path, help="a file to append pushed=true|false to, as $GITHUB_OUTPUT") + args = parser.parse_args(argv) + try: + pushed, message = record(args.repo_dir, args.branch, args.env, args.service, args.sha, args.tag, args.attempts) + except StateError as error: + print(f"::error::{error}") + return 1 + print(message) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as out: + out.write(f"pushed={'true' if pushed else 'false'}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_write_env_state.py b/tests/test_write_env_state.py new file mode 100644 index 0000000..59e464c --- /dev/null +++ b/tests/test_write_env_state.py @@ -0,0 +1,224 @@ +"""scripts/write_env_state.py against a local bare repository standing in for argocd-state.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import write_env_state # noqa: E402 + +SHA = "8b69e1a864b0f730e4552cb5616f92a9578288fa" +OTHER_SHA = "d90ad897ac744f54b0eeb69f21deefdceeda2366" +STATE = ( + "repo: https://github.com/mindsdb/scratchpad-controller.git\n" + "chart: deployment/scratchpad-controller\n" + "release: scratchpad-controller\n" + "# the image CI last built\n" + f"tag: development-head-{OTHER_SHA}\n" + f"revision: {OTHER_SHA}\n" +) + + +def git(cwd, *args): + return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, check=True).stdout + + +@pytest.fixture +def remote(tmp_path): + """A bare argocd-state with main holding dev/scratchpad-controller.yaml, and a clone of it.""" + bare = tmp_path / "argocd-state.git" + subprocess.run(["git", "init", "--quiet", "--bare", "--initial-branch=main", str(bare)], check=True) + seed = tmp_path / "seed" + subprocess.run(["git", "clone", "--quiet", str(bare), str(seed)], check=True) + git(seed, "config", "user.name", "seed") + git(seed, "config", "user.email", "seed@example.com") + git(seed, "checkout", "--quiet", "-b", "main") + (seed / "dev").mkdir() + (seed / "dev" / "scratchpad-controller.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "seed") + git(seed, "push", "--quiet", "origin", "main") + clone = tmp_path / "clone" + subprocess.run(["git", "clone", "--quiet", str(bare), str(clone)], check=True) + git(clone, "config", "user.name", "bot") + git(clone, "config", "user.email", "bot@example.com") + return bare, seed, clone + + +def main_file(bare, path="dev/scratchpad-controller.yaml"): + return subprocess.run(["git", "-C", str(bare), "show", f"main:{path}"], capture_output=True, text=True, check=True).stdout + + +class TestUpdatedText: + def test_replaces_only_the_two_lines(self): + out = write_env_state.updated_text(STATE, f"development-head-{SHA}", SHA) + assert out == STATE.replace(OTHER_SHA, SHA) + assert "# the image CI last built\n" in out + + def test_a_comment_after_the_value_stays(self): + # argocd-state's README template puts one on each of the two lines. + text = STATE.replace(f"tag: development-head-{OTHER_SHA}\n", f"tag: development-head-{OTHER_SHA} # the image tag, written by CI\n") + out = write_env_state.updated_text(text, f"development-head-{SHA}", SHA) + assert f"tag: development-head-{SHA} # the image tag, written by CI\n" in out + assert f"revision: {SHA}\n" in out + + def test_a_file_without_both_lines_is_refused(self): + with pytest.raises(write_env_state.StateError, match="no revision line"): + write_env_state.updated_text("tag: x\n", "y", SHA) + with pytest.raises(write_env_state.StateError, match="no revision or tag line"): + write_env_state.updated_text("repo: x\n", "y", SHA) + + +class TestCheck: + @pytest.mark.parametrize("field, value", [ + ("env", "Dev"), ("service", "my_service"), ("tag", "-x"), ("tag", "a b"), + ("sha", "main"), ("sha", SHA[:7]), ("sha", SHA.upper()), + ]) + def test_each_value_has_a_shape(self, field, value): + values = {"env": "dev", "service": "scratchpad-controller", "sha": SHA, "tag": f"development-head-{SHA}"} + values[field] = value + with pytest.raises(write_env_state.StateError, match=f"{field} "): + write_env_state.check(**values) + + +class TestGit: + def test_a_failed_command_is_a_state_error_with_gits_reason(self, remote): + _, _, clone = remote + with pytest.raises(write_env_state.StateError, match="git fetch failed: fatal: 'nowhere' does not appear to be a git repository"): + write_env_state.git(clone, "fetch", "nowhere") + + +class TestRecord: + def test_commits_and_pushes_the_edit(self, remote): + bare, _, clone = remote + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert pushed + assert message.startswith("dev/scratchpad-controller.yaml: tag development-head-") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + assert git(bare, "log", "-1", "--format=%s", "main") == f"dev/scratchpad-controller: development-head-{SHA}\n" + + def test_nothing_to_do_pushes_nothing(self, remote): + bare, _, clone = remote + before = git(bare, "rev-parse", "main") + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", OTHER_SHA, f"development-head-{OTHER_SHA}") + assert not pushed + assert message.endswith(f"already records development-head-{OTHER_SHA} at {OTHER_SHA}") + assert git(bare, "rev-parse", "main") == before + + def test_a_missing_file_is_refused_not_created(self, remote): + bare, _, clone = remote + with pytest.raises(write_env_state.StateError, match="dev/auth.yaml does not exist in argocd-state; adding a service is a reviewed change"): + write_env_state.record(clone, "main", "dev", "auth", SHA, "x") + assert "dev/auth.yaml" not in git(bare, "ls-tree", "-r", "--name-only", "main") + + def test_starts_from_the_remote_branch_not_the_stale_clone(self, remote): + # Another service deployed after this clone was made: both edits land. + bare, seed, clone = remote + (seed / "dev" / "auth.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "dev/auth: x") + git(seed, "push", "--quiet", "origin", "main") + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + + def test_retries_when_the_branch_moves_during_the_push(self, remote, monkeypatch): + # The remote gains a commit between this attempt's fetch and its push, as a + # concurrent deploy does; the push is rejected and the next attempt lands on top. + bare, seed, clone = remote + real_git = write_env_state.git + raced = [] + + def racing_git(repo_dir, *args, check=True): + if args[0] == "push" and not raced: + raced.append(True) + (seed / "dev" / "auth.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "dev/auth: x") + git(seed, "push", "--quiet", "origin", "main") + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", racing_git) + pushed, message = write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert pushed + assert message.endswith("(attempt 2)") + assert main_file(bare) == STATE.replace(OTHER_SHA, SHA) + assert "dev/auth.yaml" in git(bare, "ls-tree", "-r", "--name-only", "main") + + def test_gives_up_after_the_attempts(self, remote, monkeypatch): + bare, seed, clone = remote + real_git = write_env_state.git + pushes = [] + + def always_racing_git(repo_dir, *args, check=True): + if args[0] == "push": + pushes.append(True) + (seed / "dev" / f"auth{len(pushes)}.yaml").write_text(STATE) + git(seed, "add", "-A") + git(seed, "commit", "--quiet", "-m", "race") + git(seed, "push", "--quiet", "origin", "main") + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", always_racing_git) + with pytest.raises(write_env_state.StateError, match="gave up after 2 attempts"): + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}", attempts=2) + assert len(pushes) == 2 + + + def test_a_push_the_server_refuses_is_not_retried(self, remote, monkeypatch): + # A ruleset or hook saying no is final; its reason is the error, and the branch is not "moving". + bare, _, clone = remote + hook = bare / "hooks" / "pre-receive" + hook.write_text("#!/bin/sh\necho 'no deploys during the freeze' >&2\nexit 1\n") + hook.chmod(0o755) + real_git = write_env_state.git + pushes = [] + + def counting_git(repo_dir, *args, check=True): + if args[0] == "push": + pushes.append(True) + return real_git(repo_dir, *args, check=check) + + monkeypatch.setattr(write_env_state, "git", counting_git) + with pytest.raises(write_env_state.StateError, match="push to main failed: .*no deploys during the freeze"): + write_env_state.record(clone, "main", "dev", "scratchpad-controller", SHA, f"development-head-{SHA}") + assert len(pushes) == 1 + + +class TestMain: + def test_says_whether_a_commit_was_pushed(self, remote, tmp_path, capsys): + _, _, clone = remote + output = tmp_path / "github-output" + base = ["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--github-output", str(output)] + assert write_env_state.main(base + ["--sha", SHA, "--tag", f"development-head-{SHA}"]) == 0 + assert write_env_state.main(base + ["--sha", SHA, "--tag", f"development-head-{SHA}"]) == 0 + assert output.read_text() == "pushed=true\npushed=false\n" + assert "(attempt 1)" in capsys.readouterr().out + + def test_a_refusal_is_an_error_annotation_and_exit_1(self, remote, capsys): + _, _, clone = remote + rc = write_env_state.main(["--repo-dir", str(clone), "--env", "dev", "--service", "scratchpad-controller", "--sha", "main", "--tag", "x"]) + assert rc == 1 + assert capsys.readouterr().out.startswith("::error::sha 'main' is not a 40-character commit sha") + + +class TestAction: + def test_the_action_calls_the_script_with_every_input(self): + import yaml + action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) + assert set(action["inputs"]) == {"env", "service", "sha", "tag", "token", "repository", "branch", "status-token", "argocd-url"} + steps = {step.get("id") or step.get("name"): step for step in action["runs"]["steps"]} + run = steps["write"]["run"] + for flag in ("--repo-dir argocd-state", '--branch "$BRANCH"', '--env "$ENV_NAME"', '--service "$SERVICE"', '--sha "$SHA"', '--tag "$TAG"', '--github-output "$GITHUB_OUTPUT"'): + assert flag in run + + def test_the_pending_status_follows_a_pushed_commit_only(self): + import yaml + action = yaml.safe_load((Path(__file__).resolve().parents[1] / "write-env-state" / "action.yml").read_text()) + steps = action["runs"]["steps"] + pending = next(step for step in steps if step.get("name") == "Mark the commit as waiting for Argo CD") + assert steps.index(pending) > steps.index(next(step for step in steps if step.get("id") == "write")) + assert pending["if"] == "steps.write.outputs.pushed == 'true'" + assert "state=pending" in pending["run"] and 'context="argocd/$ENV_NAME"' in pending["run"] diff --git a/write-env-state/action.yml b/write-env-state/action.yml new file mode 100644 index 0000000..fcfe35c --- /dev/null +++ b/write-env-state/action.yml @@ -0,0 +1,106 @@ +# Record a deploy in mindsdb/argocd-state, where Argo CD reads what each +# long-lived environment runs: one file per service, /.yaml, and +# this sets its `revision` and `tag`. Argo CD polls that repository and syncs +# the service; nothing here touches a cluster. +# +# `sha` is the commit Argo CD renders the chart from and `tag` the image +# build-push-ecr pushed for it. On a pull_request run pass the PR's head sha +# and its `-head-` tag, not $GITHUB_SHA: that is the +# synthetic merge commit, which GitHub drops once the PR closes. +# +# `token` is an installation token for the mindsdb-release-train App, which +# argocd-state's ruleset lets push to main; mint it in the calling job with +# actions/create-github-app-token (vars.RELEASE_APP_CLIENT_ID and +# secrets.RELEASE_APP_PRIVATE_KEY). The push is retried when another deploy +# lands first. The file has to exist: adding a service is a reviewed change +# in argocd-state. scripts/write_env_state.py does the work; its tests are in +# tests/test_write_env_state.py. +# +# When a commit was pushed, commit status `argocd/` is set to pending +# on `sha` in the calling repository: Argo CD's notifier sets the same +# status to success or failure once it has synced, so a sync that never +# starts shows as a pending that never clears. A rerun that finds the file +# already up to date sets nothing, since nothing will sync. The calling job +# needs `statuses: write`. + +name: Write environment state +description: Record a service's commit and image tag in mindsdb/argocd-state for Argo CD to deploy + +inputs: + env: + description: "The environment, the directory in argocd-state (dev, staging)" + required: true + service: + description: "The service, the file name in argocd-state without .yaml" + required: true + sha: + description: "The commit Argo CD renders the chart from, 40 characters" + required: true + tag: + description: "The image tag build-push-ecr pushed for that commit" + required: true + token: + description: "An installation token for mindsdb-release-train" + required: true + repository: + description: "The state repository" + required: false + default: mindsdb/argocd-state + branch: + description: "The branch Argo CD reads" + required: false + default: main + status-token: + description: "A token that may write commit statuses on the calling repository" + required: false + default: ${{ github.token }} + argocd-url: + description: "The Argo CD that deploys this environment, for the status link" + required: false + default: https://argo.dev.mindsdb.com + +runs: + using: composite + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.branch }} + token: ${{ inputs.token }} + path: argocd-state + persist-credentials: true + + - id: write + shell: bash + env: + ENV_NAME: ${{ inputs.env }} + SERVICE: ${{ inputs.service }} + SHA: ${{ inputs.sha }} + TAG: ${{ inputs.tag }} + BRANCH: ${{ inputs.branch }} + ACTION_PATH: ${{ github.action_path }} + run: | + set -euo pipefail + git -C argocd-state config user.name "mindsdb-release-train[bot]" + git -C argocd-state config user.email "mindsdb-release-train[bot]@users.noreply.github.com" + python3 "$ACTION_PATH/../scripts/write_env_state.py" \ + --repo-dir argocd-state --branch "$BRANCH" \ + --env "$ENV_NAME" --service "$SERVICE" --sha "$SHA" --tag "$TAG" \ + --github-output "$GITHUB_OUTPUT" + + - name: Mark the commit as waiting for Argo CD + if: steps.write.outputs.pushed == 'true' + shell: bash + env: + GH_TOKEN: ${{ inputs.status-token }} + REPOSITORY: ${{ github.repository }} + ENV_NAME: ${{ inputs.env }} + SERVICE: ${{ inputs.service }} + SHA: ${{ inputs.sha }} + ARGOCD_URL: ${{ inputs.argocd-url }} + run: | + set -euo pipefail + gh api "repos/$REPOSITORY/statuses/$SHA" \ + -f state=pending -f context="argocd/$ENV_NAME" \ + -f description="Waiting for Argo CD to sync" \ + -f target_url="$ARGOCD_URL/applications/$ENV_NAME-$SERVICE?operation=true" > /dev/null