From 03a70113c6e3a0b5a4074d30c96019fd7a04dd5a Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 21:18:40 -0400 Subject: [PATCH 01/32] seed mode: make it work, and stop fetch-seed eating your store fetch-seed curled one hardcoded url and then cp'd over data.db whatever was there, so running it with local work in the store discarded it. it now takes --from a file, --url, or --url + --commit, and refuses to replace an existing store without --force. _post-start fetches a seed when there is none and we are in seed mode. the default stays disk: in seed mode the planner correctly routes a packages/ edit nowhere, so until dark edit exists there would be no way to change package code at all. said in config/dev next to the setting. two of the build-planning tests assumed disk mode without saying so and failed under seed. both pin the mode now, and each gained a seed-mode counterpart. --- config/dev | 9 ++- scripts/devcontainer/_post-start | 11 ++++ scripts/fetch-seed | 87 +++++++++++++++++++++++--- scripts/testing/test-build-planning.py | 30 +++++++-- 4 files changed, 124 insertions(+), 13 deletions(-) diff --git a/config/dev b/config/dev index db17fd6bd0..a1f0819f15 100644 --- a/config/dev +++ b/config/dev @@ -27,8 +27,13 @@ DARK_CONFIG_TRACE_SAMPLING_RULE_DEFAULT=sample-all # Turn it on for the one command you want a trace from. DARK_CONFIG_TRACE_DETAIL=off -# Package source: "disk" (reload from .dark files) or "seed" (fetch from R2) -# Default: disk (set to "seed" to use the seed-based flow) +# Package source: "disk" (reload from .dark files) or "seed" (fetch a built store). +# +# Still "disk", deliberately, and not yet the default it is headed for. In seed mode the build +# planner routes a `packages/**.dark` edit NOWHERE, which is correct -- the store is the source -- +# but it means the only way to change package code is the authoring verbs, and `dark edit +# ` does not exist yet. Flipping this before it does would leave no way to edit packages +# at all. `scripts/build/reload-packages` still works by hand if you opt in early. DARK_CONFIG_PACKAGES_SOURCE=disk ###################################### diff --git a/scripts/devcontainer/_post-start b/scripts/devcontainer/_post-start index 3f5c2f91ed..f7fb8cd63f 100755 --- a/scripts/devcontainer/_post-start +++ b/scripts/devcontainer/_post-start @@ -30,6 +30,17 @@ scripts/devcontainer/_create-app-directories scripts/devcontainer/_create-cache-directories scripts/devcontainer/_setup-hosts +# In seed mode the store IS the source, so a fresh container fetches one rather than +# producing it. Disk mode builds it from `packages/` during the build below, as always. +if [[ "${DARK_CONFIG_PACKAGES_SOURCE:-disk}" == "seed" && ! -f rundir/data.db ]]; then + log "Seed mode and no store: fetching one." + if scripts/fetch-seed >> "$LOG_FILE" 2>&1; then + log "Seed installed." + else + log "Seed fetch FAILED. Install one by hand: scripts/fetch-seed --from " + fi +fi + log "Building. Follow along: tail -F rundir/logs/build.log" if scripts/dev/build >> "$LOG_FILE" 2>&1; then log "Build succeeded. scripts/dev/status for details." diff --git a/scripts/fetch-seed b/scripts/fetch-seed index 2aa69eaa5c..0dd6f62be1 100755 --- a/scripts/fetch-seed +++ b/scripts/fetch-seed @@ -1,14 +1,87 @@ #!/usr/bin/env bash + +# Get a package seed, from a file or from a server, and install it as this rundir's store. +# +# scripts/fetch-seed --from a seed already on disk +# scripts/fetch-seed --url that server's current main +# scripts/fetch-seed --url --commit that server's main as of +# scripts/fetch-seed DARK_SEED_URL, or the built-in default +# +# A seed is ops, not projections, so the first open folds it (~seconds) and every store built +# from the same seed agrees on every id. +# +# It REFUSES to replace a store that already exists unless you pass --force. The previous +# version copied over `data.db` unconditionally, which silently discarded anything authored +# locally or pulled from a peer; that is a bad thing for a command called "fetch" to do. + set -euo pipefail -SEED_URL="https://pub-d5f652ca1b90456f85840a19eafcb700.r2.dev/darklang-seeds/seeds/seed.db" RUNDIR="${DARK_CONFIG_RUNDIR:-rundir}" +DEST_SEED="${RUNDIR}/seed.db" +DEST_DB="${RUNDIR}/${DARK_CONFIG_DB_NAME:-data.db}" + +# The bucket the old hardcoded URL pointed at. Kept as the fallback so the no-argument form +# still does something, but a real deployment passes --url or sets DARK_SEED_URL. +DEFAULT_URL="${DARK_SEED_URL:-https://pub-d5f652ca1b90456f85840a19eafcb700.r2.dev/darklang-seeds/seeds/seed.db}" + +FROM="" +URL="" +COMMIT="" +FORCE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --from) FROM="${2:?--from needs a path}"; shift 2 ;; + --url) URL="${2:?--url needs a url}"; shift 2 ;; + --commit) COMMIT="${2:?--commit needs a commit}"; shift 2 ;; + --force) FORCE=true; shift ;; + -h|--help) sed -n '3,14p' "$0" | sed 's/^# \?//'; exit 0 ;; + *) echo "fetch-seed: unknown argument $1" >&2; exit 2 ;; + esac +done + +if [[ -n "$FROM" && -n "$URL" ]]; then + echo "fetch-seed: --from and --url are alternatives, not both" >&2 + exit 2 +fi + +mkdir -p "$RUNDIR" + +if [[ -n "$FROM" ]]; then + if [[ ! -f "$FROM" ]]; then + echo "fetch-seed: no seed at $FROM" >&2 + exit 1 + fi + echo "Copying seed from $FROM" + cp "$FROM" "$DEST_SEED" +else + # `/seed/.db` is the route a server serves a pinned cut from; without a commit, its + # current main. A bare url with no path is treated as already naming the seed, which is what + # the R2 fallback is. + if [[ -n "$COMMIT" ]]; then + SEED_URL="${URL:-$DEFAULT_URL}/seed/${COMMIT}.db" + elif [[ -n "$URL" ]]; then + SEED_URL="${URL}/seed/latest.db" + else + SEED_URL="$DEFAULT_URL" + fi + + echo "Fetching seed from $SEED_URL" + curl -fL -o "$DEST_SEED" "$SEED_URL" +fi -echo "Fetching latest seed from R2..." -curl -L -o "${RUNDIR}/seed.db" "$SEED_URL" +SEED_SIZE=$(du -h "$DEST_SEED" | cut -f1) +echo "Seed at ${DEST_SEED} (${SEED_SIZE})" -# Copy as data.db so CLI/tests can grow from it -cp "${RUNDIR}/seed.db" "${RUNDIR}/data.db" +if [[ -f "$DEST_DB" && "$FORCE" != "true" ]]; then + { + echo + echo "Not installing it: ${DEST_DB} already exists." + echo "Replacing it would discard anything authored locally or pulled from a peer." + echo "Pass --force if that is what you want." + } >&2 + exit 0 +fi -SIZE=$(du -h "${RUNDIR}/data.db" | cut -f1) -echo "Seed downloaded and installed as data.db ($SIZE)" +cp "$DEST_SEED" "$DEST_DB" +echo "Installed as ${DEST_DB}. The first open folds it." diff --git a/scripts/testing/test-build-planning.py b/scripts/testing/test-build-planning.py index 5253e69c14..41d8b0252e 100755 --- a/scripts/testing/test-build-planning.py +++ b/scripts/testing/test-build-planning.py @@ -115,8 +115,20 @@ def test_fsproj_takes_a_full_build(self): self.assertTrue(self.actions("backend/src/LibDB/LibDB.fsproj").backend_full_build) def test_dark_package_reloads_packages(self): - self.assertTrue(self.actions("packages/darklang/stdlib/list.dark") - .reload_all_packages) + # Pinned to disk mode rather than inheriting the shell's. Where packages come from + # decides this answer, so a test that does not say which it means passes or fails on + # an env var the reader cannot see. + with Env(DARK_CONFIG_PACKAGES_SOURCE="disk"): + self.assertTrue(self.actions("packages/darklang/stdlib/list.dark") + .reload_all_packages) + + def test_dark_package_routes_nowhere_from_seed(self): + # The point of seed mode: the store is the source, so editing the text changes nothing + # a build should act on. + with Env(DARK_CONFIG_PACKAGES_SOURCE="seed"): + should = self.actions("packages/darklang/stdlib/list.dark") + self.assertFalse(should.reload_all_packages) + self.assertEqual(should.unrouted, ["packages/darklang/stdlib/list.dark"]) def test_migration_runs_migrations(self): self.assertTrue(self.actions("backend/migrations/001-init.sql").run_migrations) @@ -136,15 +148,25 @@ def expand(self, path, **kw): return _buildplan.expand(should, **kw).chosen() def test_fsharp_change_reaches_the_package_reload(self): - with Env(CI=None): + # Disk mode, said out loud: the cascade from migrations to the reload exists only + # because the store is built from text. + with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="disk"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), ["backend_quick_build", "run_migrations", "reload_all_packages"]) - with Env(CI="true"): + with Env(CI="true", DARK_CONFIG_PACKAGES_SOURCE="disk"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), ["backend_full_build", "run_migrations", "reload_all_packages"]) + def test_fsharp_change_stops_at_migrations_from_seed(self): + # Half the cost of an F# change today is a package reload that usually did not need to + # happen. In seed mode it does not happen at all. + with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="seed"): + self.assertEqual( + self.expand("backend/src/LibDB/Queries.fs"), + ["backend_quick_build", "run_migrations"]) + def test_full_build_replaces_the_quick_one(self): actions = self.expand("backend/src/LibDB/LibDB.fsproj") self.assertIn("backend_full_build", actions) From 6df95b3b0c726846701f6867001c2e7392e6446f Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 21:21:05 -0400 Subject: [PATCH 02/32] in seed mode, test against your store rather than the seed locally the seed is the one store guaranteed not to contain the package you just authored, so testing against it tests the wrong thing. copy data.db instead, through sqlite .backup because the store is WAL and cp drops the tail. CI keeps using the seed: it has no store of its own and wants the pinned one. --- scripts/run-backend-tests | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/run-backend-tests b/scripts/run-backend-tests index 8d620282e9..76485df032 100755 --- a/scripts/run-backend-tests +++ b/scripts/run-backend-tests @@ -116,17 +116,36 @@ cd backend && \ cd .. if [[ "${DARK_CONFIG_PACKAGES_SOURCE:-disk}" == "seed" ]]; then - # Seed mode: copy seed as test DB, Tests exe auto-grows on startup + # Seed mode: the test store is a COPY of a built store; the Tests exe folds anything + # unapplied on startup. + # + # Locally that copy is YOUR store, so a package you just authored is what the suite runs + # against. Testing the seed instead would test whatever the last fetch happened to hold, + # which is the one store guaranteed not to contain your change. + # + # CI has no store of its own to prefer, and wants the pinned one anyway: reproducible is + # the whole point of the pin. SEED_PATH="${DARK_CONFIG_RUNDIR}/seed.db" - if [[ -f "$SEED_PATH" ]]; then - echo "Copying seed.db as test database" - cp "$SEED_PATH" "$DB_PATH" - else - echo "ERROR: No seed.db found at $SEED_PATH" - echo "Run: ./scripts/run-cli export-seed /home/dark/app/rundir/seed.db" - echo "Or: ./scripts/fetch-seed" + LIVE_PATH="${DARK_CONFIG_RUNDIR}/data.db" + + SOURCE="" + if [[ ! -v CI && -f "$LIVE_PATH" ]]; then + SOURCE="$LIVE_PATH" + elif [[ -f "$SEED_PATH" ]]; then + SOURCE="$SEED_PATH" + fi + + if [[ -z "$SOURCE" ]]; then + echo "ERROR: no store to copy. Looked for $LIVE_PATH and $SEED_PATH" + echo "Run: ./scripts/fetch-seed --from " + echo "Or: ./scripts/run-cli export-seed ${DARK_CONFIG_RUNDIR}/seed.db" exit 1 fi + + # `.backup`, not `cp`: the store is WAL, so a plain copy silently omits everything still + # in `data.db-wal` -- which is exactly the work you just did. + echo "Copying $(basename "$SOURCE") as test database" + sqlite3 -cmd ".timeout 10000" "$SOURCE" ".backup '$DB_PATH'" else # Disk mode: reload packages from .dark files. # Migrations already ran inline above From 3413de7fc98afa933c0def5c9cc68bb683280178 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 21:26:28 -0400 Subject: [PATCH 03/32] dark remove: end a name nothing wrote an Unbind. a name could be bound and rebound but never ended, so one authored by mistake lived forever and greeted strangers as a constraint. one op. the content is untouched and stays reachable by hash, so callers go on working -- a reference points at content, not at a name. that is the claim the command prints, so there is a test for it: author a caller, end the callee's name, the caller still returns 4242. confirmExplicit, not confirm: a stray Return should not end a name. swept in the four shapes like every registered command. --- backend/tests/Tests/CliPackages.Tests.fs | 50 ++++++++ packages/darklang/cli/packages/remove.dark | 131 +++++++++++++++++++++ packages/darklang/cli/registry.dark | 3 +- 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 packages/darklang/cli/packages/remove.dark diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index c771333df4..a1dfcbed80 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -815,6 +815,54 @@ let aVersionMovedAndMovedBackKeepsTheLastNaming = }) +/// `remove` is the only thing that writes an `Unbind`, and the claim it prints is the +/// interesting one: the name ends, the content does not, so callers go on working. That is only +/// true because a reference points at content rather than at a name, which is the property worth +/// a test rather than a comment. +let removeEndsTheNameAndLeavesTheCallersAlone = + instanceTest "remove ends a name, and what called it still runs" (fun state -> + task { + do! start state + do! fn state "Tests.Rm.leaf" "() : Int64 = 4242L" + do! fn state "Tests.Rm.caller" "() : Int64 = Tests.Rm.leaf ()" + do! commit state "add leaf and caller" + + do! evals state "Tests.Rm.caller ()" "4242" "the caller works to begin with" + + do! run state [ "remove"; "Tests.Rm.leaf"; "-y" ] + + do! + shows + state + [ "view"; "Tests.Rm.leaf" ] + "Not found" + "the name holds nothing now" + do! + evals + state + "Tests.Rm.caller ()" + "4242" + "but the caller still runs, because it references content and not a name" + do! discardAll state + }) + +/// Bare and wrong-argument shapes, which is where this kind of command goes wrong: a confirming +/// verb that cannot find its target must refuse rather than ask about nothing. +let removeRefusesWhatIsNotThere = + instanceTest + "remove refuses a name that holds nothing, and refuses to run bare" + (fun state -> + task { + do! start state + do! + shows + state + [ "remove"; "Tests.Rm.nothingHere"; "-y" ] + "nothing here is named" + "an unknown name is named back" + do! shows state [ "remove" ] "usage: dark remove" "bare prints usage" + }) + let renameIsVisibleToEverythingThatReads = instanceTest "a renamed item is readable at its new name, by every reader" @@ -862,6 +910,8 @@ let tests : List = deleteRefusesWhatIsNotThere deprecateAndUndeprecate renameIsVisibleToEverythingThatReads + removeEndsTheNameAndLeavesTheCallersAlone + removeRefusesWhatIsNotThere aDocOnlyEditKeepsTheVersionAndStillLands aFieldsDocEditLands anEnumCasesDocEditLands diff --git a/packages/darklang/cli/packages/remove.dark b/packages/darklang/cli/packages/remove.dark new file mode 100644 index 0000000000..f92131e8c4 --- /dev/null +++ b/packages/darklang/cli/packages/remove.dark @@ -0,0 +1,131 @@ +module Darklang.Cli.Packages.Remove + +// `dark remove ` -- take a name off the shelf. +// +// One op, `Unbind`, and it is the only thing that writes one. Until now a name could be bound and +// rebound but never ENDED, so a name authored by mistake lived forever and greeted strangers as a +// constraint. +// +// What it does not do is throw anything away. The content is untouched and stays in the store, +// reachable by hash, so everything that calls it goes on working: a reference points at content, +// not at a name. What ends is the name. +// +// Not `dark delete`, which is deprecation -- that leaves the name bound and marks it obsolete, so +// callers still resolve it and are told why. Deprecation is for something that still exists and +// should not be used; removal is for a name that should not exist. + +/// The op a removal authors. +/// +/// `previous = Some hash` records what the name held, for the same reason `SetName` carries one: +/// a merge on the other side wants to know what was replaced rather than guessing. +let ops + (location: LanguageTools.ProgramTypes.PackageLocation) + (reference: LanguageTools.ProgramTypes.Reference) + : List = + [ LanguageTools.ProgramTypes.PackageOp.Unbind( + location, + Stdlib.Option.Option.Some(LanguageTools.ProgramTypes.Reference.hash reference) + ) ] + + +/// `-y` / `--yes` anywhere in the arguments, and everything that is not a flag. +let parseArgs (args: List) : (Bool * List) = + let autoConfirm = + args |> Stdlib.List.any (fun a -> (a == "-y") || (a == "--yes")) + + let rest = + args + |> Stdlib.List.filter (fun a -> + Stdlib.Bool.not ((a == "-y") || (a == "--yes"))) + + (autoConfirm, rest) + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + let (autoConfirm, rest) = parseArgs args + + match rest with + | [ name ] -> + let here = state.packageData.currentLocation + + match Packages.Location.parseRelativeTo here name with + | Error e -> + Cli.printErr $"the name to remove: {e}" + { state with exitCode = 1 } + + | Ok location -> + let dotted = + PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location + + match Packages.Rename.referenceAt state.currentBranchId location with + | None -> + Cli.printErr $"nothing here is named {dotted}." + Cli.printHint " `dark ls` lists what is; `dark search ` looks wider." + { state with exitCode = 1 } + + | Some reference -> + // `confirmExplicit`, not `confirm`: a stray Return should not end a name. An unattended + // run says `-y`, which is the only thing that means yes without a person. + let proceed = + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + $"Remove the name {dotted}? Its content stays and its callers keep working. (y/n): ") + + if proceed then + match SCM.PackageOps.add state.currentBranchId (ops location reference) with + | Ok _ -> + Cli.printOk $"removed the name {dotted}." + + Cli.printHint + " the content is still in the store and its callers still resolve it: they reference content, not names." + + Cli.printHint " `dark status` shows the change; it travels when you sync." + + state + + | Error e -> + Cli.printErr $"couldn't remove: {e}" + { state with exitCode = 1 } + else + Cli.printOk "left it alone." + state + + | [] -> + Cli.printErr "usage: dark remove [-y]" + Cli.printHint " the name is relative to where you are (`dark nav` moves)." + { state with exitCode = 1 } + + | _ -> + Cli.printErr "usage: dark remove [-y]" + Cli.printHint " one name at a time, so the confirmation can say what it is ending." + { state with exitCode = 1 } + + +let help (_state: Cli.AppState) : String = + [ "" + " dark remove [-y]" + "" + " Ends a name. The content it held is untouched: it stays in the store," + " reachable by hash, and everything that calls it goes on working, because" + " a reference points at content rather than at a name." + "" + " Use it when the NAME was the mistake. For something that still exists and" + " should not be used, `dark delete` deprecates instead, which leaves the name" + " bound and tells callers why." + "" + " One `Unbind` op is authored, so `dark status` shows it and it travels when" + " you sync." + "" + " -y, --yes skip the confirmation; required for an unattended run" ] + |> Stdlib.String.join "\n" + + +let complete + (state: Cli.AppState) + (args: List) + : List = + match args with + | [] -> Packages.Nav.complete state args + | [ _ ] -> Packages.Nav.complete state args + | _ -> [] diff --git a/packages/darklang/cli/registry.dark b/packages/darklang/cli/registry.dark index e3b3f643ae..f1380cfc3a 100644 --- a/packages/darklang/cli/registry.dark +++ b/packages/darklang/cli/registry.dark @@ -40,6 +40,7 @@ module Registry = ("module", "Define multiple package declarations", [], Packages.ModuleCommand.execute, Packages.ModuleCommand.help, Packages.ModuleCommand.complete) ("edit", "Change an item without retyping it", [], Packages.Edit.execute, Packages.Edit.help, Packages.Edit.complete) ("rename", "Give an item a different name (its content and callers are untouched)", [], Packages.Rename.execute, Packages.Rename.help, Packages.Rename.complete) + ("remove", "End a name (the content stays, and its callers keep working)", [], Packages.Remove.execute, Packages.Remove.help, Packages.Remove.complete) ("hash", "Show hash of a package item", [], Packages.Hash.execute, Packages.Hash.help, Packages.Hash.complete) // SCM commands ("status", "What's in your draft, in one line", [ "wip", "changes" ], Cli.Status.execute, Cli.Status.help, Cli.Status.complete) @@ -197,7 +198,7 @@ module Registry = // Command groups organized by category (shared between compact and detailed views) let commandGroups () : List<(String * List)> = - [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "hash", "db", "deprecate", "delete", "undeprecate" ]) + [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "hash", "db", "deprecate", "delete", "undeprecate" ]) ("Changes", [ "status", "commit", "commits", "show", "discard", "undo", "propagate", "conflicts", "constraints", "ack", "ops" ]) ("Branches", [ "branch", "branches", "switch", "merge", "rebase", "diff", "log", "resolve" ]) From e957ea2599862eaad0485fab7a1c6c7929cb4dc0 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 22:06:28 -0400 Subject: [PATCH 04/32] dark grep: search bodies, not names after the flip there is no packages/ tree to grep, so this is the replacement. search looks at names; this looks at source. nothing stores source: the store holds ops and text comes from the pretty printer. so it is cached, keyed by content hash, which never needs invalidating because content is immutable. the cache lives beside the store like credentials.db -- it is derived and per-install, and Seed.export strips by a denylist, so a table added to the store would ride out in every seed. takes an optional module to search under. unscoped it renders everything the first time, which is minutes; scoped it is the common case and it is quick. it searches RENDERED source, so /// docs are in and // asides are not: the parser does not keep them. said in --help, because a search tool you cannot trust a negative answer from is worse than none -- which is also why it now reports items it could not render instead of counting them as non-matches. --- backend/tests/Tests/CliPackages.Tests.fs | 78 +++++++ packages/darklang/cli/packages/grep.dark | 262 +++++++++++++++++++++++ packages/darklang/cli/registry.dark | 3 +- 3 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 packages/darklang/cli/packages/grep.dark diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index a1dfcbed80..53db247486 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -819,6 +819,81 @@ let aVersionMovedAndMovedBackKeepsTheLastNaming = /// interesting one: the name ends, the content does not, so callers go on working. That is only /// true because a reference points at content rather than at a name, which is the property worth /// a test rather than a comment. +/// `grep` searches BODIES, which is the half `search` does not do. Scoped to one module on +/// purpose: unscoped it renders every live item, and the point of the scope argument is that a +/// test, like a person, usually knows roughly where to look. +let grepFindsSourceAndNotJustNames = + instanceTest + "grep matches a function body, and reports name and line" + (fun state -> + task { + do! start state + // A string LITERAL, not a comment: grep reads rendered source, and the renderer prints + // the AST, which keeps `///` docs and drops `//` asides. + do! fn state "Tests.Grep.needle" "() : String = \"findmeplease\"" + do! fn state "Tests.Grep.other" "() : String = \"something else\"" + do! commit state "grep fixture" + + do! + shows + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "Tests.Grep.needle:" + "the hit names the item" + do! + shows + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "findmeplease" + "and shows the matching line" + do! + lacks + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "Tests.Grep.other" + "an item whose body does not match is not reported" + do! discardAll state + }) + +/// The cache is the feature, so the second run has to agree with the first. Keyed by content +/// hash, which is why it never needs invalidating. +let grepAgreesWithItselfOnceCached = + instanceTest "a second grep, reading the cache, finds the same thing" (fun state -> + task { + do! start state + do! fn state "Tests.GrepTwice.f" "() : String = \"cachedtoken\"" + do! commit state "grep cache fixture" + + do! + shows + state + [ "grep"; "cachedtoken"; "Tests.GrepTwice" ] + "Tests.GrepTwice.f:" + "cold" + do! + shows + state + [ "grep"; "cachedtoken"; "Tests.GrepTwice" ] + "Tests.GrepTwice.f:" + "warm" + do! discardAll state + }) + +/// A search tool you cannot trust a negative answer from is worse than none, so "no hits" and +/// "could not read it" must not look alike. +let grepSaysWhenItFindsNothing = + instanceTest "grep says so when nothing matches" (fun state -> + task { + do! start state + do! + shows + state + [ "grep"; "zzz-not-in-any-source-zzz"; "Darklang.Stdlib.Option" ] + "no live item's source contains" + "an honest empty answer" + do! shows state [ "grep" ] "usage: dark grep" "bare prints usage" + }) + let removeEndsTheNameAndLeavesTheCallersAlone = instanceTest "remove ends a name, and what called it still runs" (fun state -> task { @@ -912,6 +987,9 @@ let tests : List = renameIsVisibleToEverythingThatReads removeEndsTheNameAndLeavesTheCallersAlone removeRefusesWhatIsNotThere + grepFindsSourceAndNotJustNames + grepAgreesWithItselfOnceCached + grepSaysWhenItFindsNothing aDocOnlyEditKeepsTheVersionAndStillLands aFieldsDocEditLands anEnumCasesDocEditLands diff --git a/packages/darklang/cli/packages/grep.dark b/packages/darklang/cli/packages/grep.dark new file mode 100644 index 0000000000..fb769ab6fe --- /dev/null +++ b/packages/darklang/cli/packages/grep.dark @@ -0,0 +1,262 @@ +module Darklang.Cli.Packages.Grep + +// `dark grep ` -- search the SOURCE of what is live on your branch. +// +// `dark search` looks at names. This looks at bodies, which is the one you want when you remember +// what code DID and not what it was called. After the flip there is no `packages/` tree to reach +// for instead, so this is the replacement for it. +// +// What it searches is RENDERED source, printed from the stored AST, so `///` docs are in and `//` +// asides are not -- the parser does not keep them. Worth knowing before concluding a string is +// absent. +// +// The source is not stored anywhere. The store holds ops; text is produced by the pretty-printer, +// about 9ms an item across roughly six and a half thousand of them, so grepping by rendering +// everything on demand would take a minute. It is cached instead, keyed by CONTENT HASH, which +// never needs invalidating because content is immutable: an item is rendered once per hash, ever. +// +// The cache lives BESIDE the store rather than in it, the way `credentials.db` does. It is derived, +// it is per-install, and it has no business travelling in a seed or a sync -- `Seed.export` strips +// by a denylist, so a table added to the store would ride out in every seed until someone +// remembered to name it. + +/// Where the rendered-source cache lives: beside the store, never inside it. +let cachePath () : String = + let storeDir = Stdlib.Cli.Path.parent (Stdlib.LocalStore.path ()) + Stdlib.Cli.Path.join [ storeDir, "source-cache.db" ] + + +/// Create the cache if this install has never grepped. Cheap enough to call every time. +let ensureCache () : Unit = + let sql = + "CREATE TABLE IF NOT EXISTS source_cache_v0 (item_hash TEXT PRIMARY KEY, source TEXT NOT NULL)" + + match Stdlib.Sqlite.exec (cachePath ()) sql with + | Ok _ -> () + | Error e -> + Cli.printErr $"could not open the source cache at {cachePath ()}: {e}" + () + + +/// Every live name on this branch, as (hash, owner, modules, name). Straight SQL because this is a +/// listing of six thousand rows and the search API materialises whole definitions to produce one. +/// +/// `source != 'unbind'` drops tombstones: an Unbind writes a row that is unlisted from birth, so it +/// is history rather than something you can read. +let liveItems (scope: String) : List<(String * String * String * String * String)> = + // `item_type` matters: without it the renderer has to try fn, then type, then value, and pay a + // store query for each guess. The column already knows. + let selectFrom = + "SELECT item_hash, owner, modules, name, item_type FROM locations + WHERE unlisted_at IS NULL AND source != 'unbind'" + + let ordering = " ORDER BY owner, modules, name" + + let result = + if scope == "" then + Stdlib.Sqlite.query (Stdlib.LocalStore.path ()) (selectFrom ++ ordering) + else + // The scope is a dotted prefix of `owner.modules`: the module itself, or anything under it. + let sql = + selectFrom + ++ " AND ((owner || '.' || modules) = @p0 OR (owner || '.' || modules) LIKE @p1)" + ++ ordering + + Stdlib.Sqlite.queryP + (Stdlib.LocalStore.path ()) + sql + [ scope, $"{scope}.%" ] + + match result with + | Error _ -> [] + | Ok rows -> + rows + |> Stdlib.List.filterMap (fun row -> + let get = fun col -> Stdlib.Sqlite.textField row col + + match + (get "item_hash", get "owner", get "modules", get "name", get "item_type") + with + | (Some h, Some o, Some m, Some n, Some t) -> + Stdlib.Option.Option.Some((h, o, m, n, t)) + | _ -> Stdlib.Option.Option.None) + + +/// Everything this install has already rendered, as hash -> source. +/// +/// One query, not one per item. A grep over the whole store asks about six and a half thousand +/// hashes, and asking SQLite six and a half thousand times costs more than the rendering the +/// cache exists to avoid. +let cachedAll () : Dict = + match Stdlib.Sqlite.query (cachePath ()) "SELECT item_hash, source FROM source_cache_v0" with + | Error _ -> Stdlib.Dict.empty + | Ok rows -> + rows + |> Stdlib.List.fold (Stdlib.Dict.empty) (fun acc row -> + match + (Stdlib.Sqlite.textField row "item_hash", Stdlib.Sqlite.textField row "source") + with + | (Some h, Some src) -> Stdlib.Dict.set acc h src + | _ -> acc) + + +let remember (hash: String) (source: String) : Unit = + let sql = + "INSERT OR REPLACE INTO source_cache_v0 (item_hash, source) VALUES (@p0, @p1)" + + let _ = Stdlib.Sqlite.execP (cachePath ()) sql [ hash, source ] + () + + +/// A dotted name, for reporting a hit. +let dotted (owner: String) (modules: String) (name: String) : String = + if modules == "" then $"{owner}.{name}" else $"{owner}.{modules}.{name}" + + +/// The source of one item, from the cache or by rendering it. +let sourceOf + (branchId: Uuid) + (warm: Dict) + (hash: String) + (owner: String) + (modules: String) + (name: String) + (itemType: String) + : Stdlib.Option.Option = + match Stdlib.Dict.get warm hash with + | Some s -> Stdlib.Option.Option.Some s + | None -> + let modulePath = + if modules == "" then [] else Stdlib.String.split modules "." + + let location = + LanguageTools.ProgramTypes.PackageLocation + { owner = owner; modules = modulePath; name = name } + + let wrapped = + match itemType with + | "fn" -> Packages.PackageLocation.Function location + | "type" -> Packages.PackageLocation.Type location + | _ -> Packages.PackageLocation.Value location + + match Packages.View.rawSource branchId wrapped with + | Ok s -> + let _ = remember hash s + Stdlib.Option.Option.Some s + | Error _ -> Stdlib.Option.Option.None + + +/// The matching lines of one item's source, as (lineNumber, text). +let matchesIn (source: String) (pattern: String) : List<(Int * String)> = + source + |> Stdlib.String.split "\n" + |> Stdlib.List.indexedMap (fun i line -> (i + 1, line)) + |> Stdlib.List.filter (fun (_i, line) -> Stdlib.String.contains line pattern) + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + let (pattern, scope) = + match args with + | [ p ] -> (p, "") + | [ p, sc ] -> (p, sc) + | _ -> ("", "") + + if pattern == "" then + Cli.printErr "usage: dark grep [module]" + + Cli.printHint + " with no module it searches everything live on your branch, which is slow the first time." + + { state with exitCode = 1 } + else + + let _ = ensureCache () + let items = liveItems scope + let warm = cachedAll () + let total = Stdlib.List.length items + + // Said once, before the wait rather than after it: the first grep on an install renders + // everything, and a minute of silence reads as a hang. + let anyCached = + match items with + | (h, _, _, _, _) :: _ -> (Stdlib.Dict.get warm h) != Stdlib.Option.Option.None + | [] -> true + + if Stdlib.Bool.not anyCached then + Cli.printHint + $" first grep on this install: rendering {Stdlib.Int.toString total} items. Later greps read the cache." + + // Rendered per item, keeping the failures rather than dropping them: an item whose source + // cannot be produced is NOT an item without matches, and a search tool that cannot tell you + // the difference is one you cannot trust a negative answer from. + let outcomes = + items + |> Stdlib.List.map (fun (hash, owner, modules, name, itemType) -> + (dotted owner modules name, + sourceOf state.currentBranchId warm hash owner modules name itemType)) + + let unreadable = + outcomes + |> Stdlib.List.filter (fun (_where, source) -> + match source with + | None -> true + | Some _ -> false) + + let hits = + outcomes + |> Stdlib.List.map (fun (where, source) -> + match source with + | None -> [] + | Some s -> + matchesIn s pattern + |> Stdlib.List.map (fun (lineNo, line) -> + $"{where}:{Stdlib.Int.toString lineNo}: {Stdlib.String.trim line}")) + |> Stdlib.List.flatten + + let unreadableCount = Stdlib.List.length unreadable + + if unreadableCount > 0 then + Cli.printErr + $"could not render {Stdlib.Int.toString unreadableCount} of {Stdlib.Int.toString total} items; they were NOT searched." + + unreadable + |> Stdlib.List.take 3 + |> Stdlib.List.map (fun (where, _s) -> $" {where}") + |> Stdlib.printLines + + match hits with + | [] -> + Cli.printOk $"no live item's source contains {pattern}." + state + | _ -> + hits |> Stdlib.printLines + Cli.printHint + $" {Stdlib.Int.toString (Stdlib.List.length hits)} line(s). `dark search` looks at names instead." + + state + + + +let help (_state: Cli.AppState) : String = + [ "" + " dark grep " + "" + " Searches the SOURCE of everything live on your branch, and prints" + " name:line: for each hit. `dark search` searches names instead." + "" + " Plain substring matching, not a regex." + "" + " It searches RENDERED source, which is printed from the stored AST. So" + " `///` docs are searchable and `//` asides are not: those are not kept." + "" + " The first run on an install renders every item and caches the result" + " beside the store, keyed by content hash. Later runs read the cache, and" + " because content is immutable it never needs invalidating." ] + |> Stdlib.String.join "\n" + + +let complete + (_state: Cli.AppState) + (_args: List) + : List = + [] diff --git a/packages/darklang/cli/registry.dark b/packages/darklang/cli/registry.dark index f1380cfc3a..df24b4b4af 100644 --- a/packages/darklang/cli/registry.dark +++ b/packages/darklang/cli/registry.dark @@ -31,6 +31,7 @@ module Registry = ("scripts", "Store, manage, and run Dark scripts", [], Cli.Scripts.execute, Cli.Scripts.help, Cli.Scripts.complete) ("view", "View details of a module, type, value, or fns", [], Packages.View.execute, Packages.View.help, Packages.View.complete) ("tree", "Display package hierarchy in tree format", [ "packages" ], Packages.Tree.execute, Packages.Tree.help, Packages.Tree.complete) + ("grep", "Search the source of what is live (`search` looks at names)", [], Packages.Grep.execute, Packages.Grep.help, Packages.Grep.complete) ("search", "Search modules, types, functions, and values", [], Packages.Search.execute, Packages.Search.help, Packages.Search.complete) ("typecheck", "Audit package declarations without changing them", [], Packages.Typecheck.execute, Packages.Typecheck.help, Packages.Typecheck.complete) ("deps", "Show dependencies and dependents", [ "dependencies" ], Cli.Deps.execute, Cli.Deps.help, Cli.Deps.complete) @@ -198,7 +199,7 @@ module Registry = // Command groups organized by category (shared between compact and detailed views) let commandGroups () : List<(String * List)> = - [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "hash", "db", "deprecate", "delete", "undeprecate" ]) + [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "grep", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "hash", "db", "deprecate", "delete", "undeprecate" ]) ("Changes", [ "status", "commit", "commits", "show", "discard", "undo", "propagate", "conflicts", "constraints", "ack", "ops" ]) ("Branches", [ "branch", "branches", "switch", "merge", "rebase", "diff", "log", "resolve" ]) From be22c082f5b79aeb683daa2e7ad8ebe009131126 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 22:10:55 -0400 Subject: [PATCH 05/32] grep refuses an unscoped cold search instead of hanging for minutes a new command joins the registry sweep the day it is registered, and the sweep runs everything with a bogus argument. unscoped grep renders 6,604 items, so adding it quietly cost nine minutes of suite time. it now says what the search would cost and how to avoid it: name a module, or pass --all to accept the wait. the sweep takes the refusal in a second. sweep 9m48 -> 59s --- packages/darklang/cli/packages/grep.dark | 29 ++++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/darklang/cli/packages/grep.dark b/packages/darklang/cli/packages/grep.dark index fb769ab6fe..bd1f639df0 100644 --- a/packages/darklang/cli/packages/grep.dark +++ b/packages/darklang/cli/packages/grep.dark @@ -155,17 +155,23 @@ let matchesIn (source: String) (pattern: String) : List<(Int * String)> = let execute (state: Cli.AppState) (args: List) : Cli.AppState = + let acceptedTheWait = + args |> Stdlib.List.any (fun a -> a == "--all") + + let positional = + args |> Stdlib.List.filter (fun a -> a != "--all") + let (pattern, scope) = - match args with + match positional with | [ p ] -> (p, "") | [ p, sc ] -> (p, sc) | _ -> ("", "") if pattern == "" then - Cli.printErr "usage: dark grep [module]" + Cli.printErr "usage: dark grep [module] [--all]" Cli.printHint - " with no module it searches everything live on your branch, which is slow the first time." + " a module scopes the search and is much quicker; --all accepts the wait on a cold cache." { state with exitCode = 1 } else @@ -182,9 +188,22 @@ let execute (state: Cli.AppState) (args: List) : Cli.AppState = | (h, _, _, _, _) :: _ -> (Stdlib.Dict.get warm h) != Stdlib.Option.Option.None | [] -> true + // An unscoped grep against a cold cache renders everything, which is minutes. Refused rather + // than done silently: a command that hangs for five minutes without saying why is one people + // stop trusting, and this one is also run by the registry sweep, where it cost nine minutes + // of the suite before it asked first. + if (scope == "") && (Stdlib.Bool.not anyCached) && (Stdlib.Bool.not acceptedTheWait) then + Cli.printErr + $"searching everything means rendering {Stdlib.Int.toString total} items, which takes minutes on this install." + + Cli.printHint " `dark grep ` searches under one module and is quick." + Cli.printHint " `dark grep --all` does it anyway; the wait is paid once, then cached." + { state with exitCode = 1 } + else + if Stdlib.Bool.not anyCached then Cli.printHint - $" first grep on this install: rendering {Stdlib.Int.toString total} items. Later greps read the cache." + $" rendering {Stdlib.Int.toString total} items. This is paid once; later greps read the cache." // Rendered per item, keeping the failures rather than dropping them: an item whose source // cannot be produced is NOT an item without matches, and a search tool that cannot tell you @@ -239,7 +258,7 @@ let execute (state: Cli.AppState) (args: List) : Cli.AppState = let help (_state: Cli.AppState) : String = [ "" - " dark grep " + " dark grep [module] [--all]" "" " Searches the SOURCE of everything live on your branch, and prints" " name:line: for each hit. `dark search` searches names instead." From 8eb062f15df1af6100f7847bd721481d568d35b6 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 22:23:14 -0400 Subject: [PATCH 06/32] grep reads through the branch overlay, not straight from locations a guard test caught this: locations is main's projection, so grep enumerating it directly searched MAIN while standing on a branch, and said nothing was wrong. exactly the class the SCM silo exists to stop. the enumerating read belongs in the silo beside liveBindingFor, so allLiveBindings lays the chain overlay over main: a branch's SetName replaces main's entry, and a name the chain unbound is dropped even though main holds it. the main-scoped half is marked as the silo's rule requires. tested both directions: a branch finds its own work, main does not. --- backend/tests/Tests/CliPackages.Tests.fs | 29 +++++++++ packages/darklang/cli/packages/grep.dark | 59 ++++++----------- packages/darklang/scm/packageOpsBindings.dark | 63 +++++++++++++++++++ 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index 53db247486..e6ffc8587e 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -881,6 +881,34 @@ let grepAgreesWithItselfOnceCached = /// A search tool you cannot trust a negative answer from is worse than none, so "no hits" and /// "could not read it" must not look alike. +/// The bug the SCM silo's guard exists to stop, in grep's shape: `locations` is main's +/// projection, so an enumeration that read it directly would search MAIN from a branch and report +/// nothing wrong. This is the test that the overlay is actually consulted. +let grepSeesTheBranchYouAreStandingOn = + instanceTest "grep finds a branch's own work, and main does not" (fun state -> + task { + do! start state + do! switch state "grep-branch" + do! fn state "Tests.GrepBranch.only" "() : String = \"branchonlytoken\"" + + do! + shows + state + [ "grep"; "branchonlytoken"; "Tests.GrepBranch" ] + "Tests.GrepBranch.only:" + "the branch sees its own work" + + do! onMain state + do! + shows + state + [ "grep"; "branchonlytoken"; "Tests.GrepBranch" ] + "no live item's source contains" + "main does not" + + do! archiveBranches state [ "grep-branch" ] + }) + let grepSaysWhenItFindsNothing = instanceTest "grep says so when nothing matches" (fun state -> task { @@ -990,6 +1018,7 @@ let tests : List = grepFindsSourceAndNotJustNames grepAgreesWithItselfOnceCached grepSaysWhenItFindsNothing + grepSeesTheBranchYouAreStandingOn aDocOnlyEditKeepsTheVersionAndStillLands aFieldsDocEditLands anEnumCasesDocEditLands diff --git a/packages/darklang/cli/packages/grep.dark b/packages/darklang/cli/packages/grep.dark index bd1f639df0..b5d9f372b7 100644 --- a/packages/darklang/cli/packages/grep.dark +++ b/packages/darklang/cli/packages/grep.dark @@ -38,48 +38,25 @@ let ensureCache () : Unit = () -/// Every live name on this branch, as (hash, owner, modules, name). Straight SQL because this is a -/// listing of six thousand rows and the search API materialises whole definitions to produce one. +/// Every live name on this branch, as (hash, owner, modules, name, itemType). /// -/// `source != 'unbind'` drops tombstones: an Unbind writes a row that is unlisted from birth, so it -/// is history rather than something you can read. -let liveItems (scope: String) : List<(String * String * String * String * String)> = - // `item_type` matters: without it the renderer has to try fn, then type, then value, and pay a - // store query for each guess. The column already knows. - let selectFrom = - "SELECT item_hash, owner, modules, name, item_type FROM locations - WHERE unlisted_at IS NULL AND source != 'unbind'" - - let ordering = " ORDER BY owner, modules, name" - - let result = - if scope == "" then - Stdlib.Sqlite.query (Stdlib.LocalStore.path ()) (selectFrom ++ ordering) +/// Through the SCM silo, NOT a direct read of `locations`: that table is main's projection, so a +/// branch's own work is not in it and a grep run from a branch would search main and say nothing +/// was wrong. `allLiveBindings` lays the chain overlay over main, which is the branch-aware answer. +let liveItems (branchId: Uuid) (scope: String) : List<(String * String * String * String * String)> = + SCM.PackageOps.allLiveBindings branchId + |> Stdlib.List.filterMap (fun b -> + let dottedModule = if b.modules == "" then b.owner else $"{b.owner}.{b.modules}" + + let inScope = + (scope == "") + || (dottedModule == scope) + || (Stdlib.String.startsWith dottedModule $"{scope}.") + + if inScope then + Stdlib.Option.Option.Some((b.hash, b.owner, b.modules, b.name, b.itemType)) else - // The scope is a dotted prefix of `owner.modules`: the module itself, or anything under it. - let sql = - selectFrom - ++ " AND ((owner || '.' || modules) = @p0 OR (owner || '.' || modules) LIKE @p1)" - ++ ordering - - Stdlib.Sqlite.queryP - (Stdlib.LocalStore.path ()) - sql - [ scope, $"{scope}.%" ] - - match result with - | Error _ -> [] - | Ok rows -> - rows - |> Stdlib.List.filterMap (fun row -> - let get = fun col -> Stdlib.Sqlite.textField row col - - match - (get "item_hash", get "owner", get "modules", get "name", get "item_type") - with - | (Some h, Some o, Some m, Some n, Some t) -> - Stdlib.Option.Option.Some((h, o, m, n, t)) - | _ -> Stdlib.Option.Option.None) + Stdlib.Option.Option.None) /// Everything this install has already rendered, as hash -> source. @@ -177,7 +154,7 @@ let execute (state: Cli.AppState) (args: List) : Cli.AppState = else let _ = ensureCache () - let items = liveItems scope + let items = liveItems state.currentBranchId scope let warm = cachedAll () let total = Stdlib.List.length items diff --git a/packages/darklang/scm/packageOpsBindings.dark b/packages/darklang/scm/packageOpsBindings.dark index cbbcd2b576..db2d9e78a4 100644 --- a/packages/darklang/scm/packageOpsBindings.dark +++ b/packages/darklang/scm/packageOpsBindings.dark @@ -233,6 +233,69 @@ let chainBindings (branchId: Uuid) : List = chainBindingEntries branchId |> Stdlib.List.filterMap (fun w -> w.binding) +/// EVERY name live on , as bindings. The enumerating form of . +/// +/// Main's projection, then the chain overlay laid over it: a branch's SetName replaces main's entry +/// for that name, and a name the chain left unbound is dropped even though main still holds it. +/// +/// Here rather than at the call site because the overlay is the whole point. A caller that wanted +/// "everything live" and read `locations` would get MAIN's list while standing on a branch, and get +/// it plausibly -- which is the class of bug this silo exists to stop. +let allLiveBindings (branchId: Uuid) : List = + let mainRows = + // main-scoped: deliberately main's projection, as the BASE the chain overlay below replaces + // entries in. Reading it alone would be the bug; reading it as the floor is the design. + let sql = + "SELECT owner, modules, name, item_type, item_hash, COALESCE(origin_ts, '') AS origin_ts + FROM locations + WHERE unlisted_at IS NULL AND source != 'unbind'" + + match Stdlib.Sqlite.query (Stdlib.LocalStore.path ()) sql with + | Error _ -> [] + | Ok rows -> + rows + |> Stdlib.List.filterMap (fun row -> + let get (col: String) : Stdlib.Option.Option = + Stdlib.Sqlite.textField row col + + match + (get "owner", get "modules", get "name", get "item_type", get "item_hash") + with + | (Some o, Some m, Some n, Some t, Some h) -> + Stdlib.Option.Option.Some( + Conflicts.Binding + { owner = o + modules = m + name = n + itemType = t + hash = h + originTs = (get "origin_ts") |> Stdlib.Option.withDefault "" + author = "" + previous = Stdlib.Option.Option.None } + ) + | _ -> Stdlib.Option.Option.None) + + if SCM.Branch.isMain branchId then + mainRows + else + let words = chainBindingEntries branchId + + let key (o: String) (m: String) (n: String) : String = $"{o}|{m}|{n}" + + let spokenFor = + words + |> Stdlib.List.fold Stdlib.Dict.empty (fun acc w -> + Stdlib.Dict.set acc (key w.owner w.modules w.name) true) + + // Main's entries that the chain has NOT rebound or unbound, plus whatever the chain leaves bound. + let untouched = + mainRows + |> Stdlib.List.filter (fun b -> + (Stdlib.Dict.get spokenFor (key b.owner b.modules b.name)) == Stdlib.Option.Option.None) + + Stdlib.List.append untouched (words |> Stdlib.List.filterMap (fun w -> w.binding)) + + /// What resolves to on : the chain overlay first, then main's projection, /// which is where every chain ends. THE branch-aware read of a live binding. /// From b89fa7795c58cb7dca0ec411e73d86edaf29b568 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Fri, 11 Sep 2026 22:29:50 -0400 Subject: [PATCH 07/32] resolve the kernel's fn refs by name, behind a flag f# only CALLS those seventeen; it never takes one apart. so the frozen contract is the name and the signature, and the newest committed binding can win -- which is what would make the store the source for the cli's own entry points rather than for everything except them. types stay pinned: a DRecord the kernel builds carries its type's hash. the hook lives in PackageRefs and is installed by LibDB, because LibDB depends on LibExecution and not the reverse. main's committed projection only: resolving entry points through a branch overlay would run a branch's parser the moment you stood on one. OFF unless DARK_REFS_BY_NAME=1, and it falls back to the pinned hash for any name the store does not bind, so with the flag off this changes nothing. it also does nothing useful yet: Darklang is a reserved owner, so nothing can rebind those names anyway. that is the same question as who may fold on the server, and it is written up in the worklist. --- backend/src/LibDB/PackageManager.fs | 38 ++++++++++++ backend/src/LibExecution/PackageRefs.fs | 79 ++++++++++++++++++------- 2 files changed, 96 insertions(+), 21 deletions(-) diff --git a/backend/src/LibDB/PackageManager.fs b/backend/src/LibDB/PackageManager.fs index 64c5d72ddc..398dd00b9d 100644 --- a/backend/src/LibDB/PackageManager.fs +++ b/backend/src/LibDB/PackageManager.fs @@ -1,6 +1,8 @@ module LibDB.PackageManager open Prelude +open Fumble +open LibDB.Sqlite open LibExecution.ProgramTypes module RT = LibExecution.RuntimeTypes @@ -630,6 +632,42 @@ let mutable private currentBranchIdOpt : Option = None /// Delta ops for branches OTHER than the active one, loaded on demand. Bounded by how many /// branches a process actually asks about, which for a CLI is one or two. +/// Teach `PackageRefs` to resolve the kernel's seventeen FN refs against this store by NAME. +/// +/// Installed here because `PackageRefs` lives in `LibExecution`, which knows nothing about a +/// store: `LibDB` depends on it, not the reverse, so the dependency has to be handed down rather +/// than reached for. +/// +/// Main's committed projection only, and by design. These are the entry points the kernel calls +/// into Dark through, so resolving them through a branch overlay would mean the binary ran a +/// branch's parser the moment you stood on one. Returns `None` for anything not bound here, and +/// `PackageRefs` falls back to the pinned hash, so a store that predates a ref behaves as before. +/// +/// Off unless `DARK_REFS_BY_NAME=1`; see the note on `PackageRefs.resolveFnByName`. +let private resolveKernelFnByName + (modules : string list) + (name : string) + : string option = + try + let modulesStr = String.concat "." modules + + Sql.query + "SELECT item_hash + FROM locations + WHERE owner = 'Darklang' AND modules = @modules AND name = @name + AND item_type = 'fn' AND unlisted_at IS NULL AND source != 'unbind' + LIMIT 1" + |> Sql.parameters [ "modules", Sql.string modulesStr; "name", Sql.string name ] + |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") + |> fun t -> t.Result + with _ -> + // No store yet, or one without the table: migrations run before any of this exists, and a + // ref resolved during them must fall back rather than fail the boot. + None + +LibExecution.PackageRefs.resolveFnByName <- resolveKernelFnByName + + let private otherBranchOps = System.Collections.Concurrent.ConcurrentDictionary>() diff --git a/backend/src/LibExecution/PackageRefs.fs b/backend/src/LibExecution/PackageRefs.fs index c7d980fee3..68552b8eb9 100644 --- a/backend/src/LibExecution/PackageRefs.fs +++ b/backend/src/LibExecution/PackageRefs.fs @@ -137,6 +137,28 @@ let setHashes (hashes : Map) : unit = hashGeneration <- hashGeneration + 1 +/// How a FN ref resolves against the live store, when it does. +/// +/// Installed by whoever owns the store, because `LibDB` depends on `LibExecution` and not the +/// other way round: `PackageRefs` cannot read `locations` itself. `None` until installed, and +/// `None` for a name the store does not bind. +/// +/// Why fns and not types. F# only CALLS these seventeen; it never takes one apart. So the frozen +/// contract is the name and the signature, and the newest committed binding may win -- which is +/// what makes "the store is the source" true of the CLI's own entry points, rather than true of +/// everything except them. A TYPE is different and must stay pinned: a `DRecord` the kernel +/// builds carries its type's hash, so a store whose newest version of that type has a different +/// shape would hand the kernel a value it cannot read. +let mutable resolveFnByName : (string list -> string -> string option) = + fun _ _ -> None + +/// Off by default. By-name resolution means the binary runs whatever the store currently binds +/// `Cli.executeCliCommand` to, which is the intent -- and also means a store with a broken entry +/// point produces a CLI that cannot start. Opt in with `DARK_REFS_BY_NAME=1` until that trade has +/// been made deliberately. +let private byNameEnabled : Lazy = + lazy (System.Environment.GetEnvironmentVariable "DARK_REFS_BY_NAME" = "1") + /// Shared body of `Type.p` and `Fn.p`: a closure resolving `/.` /// against the hash file. Resolution is cached once per hash generation: the answer /// cannot change while the generation is stable, and resolving per call costs an @@ -154,27 +176,42 @@ let private makeRef let mutable cached = "" fun () -> - let gen = currentGeneration () - if gen = cachedGen then - cached - else - let fqn = $"""{kind}/{String.concat "." modules}.{name}""" - let h = getHashes () - match Map.tryFind fqn h with - | Some hash -> - record hash - cachedGen <- gen - cached <- hash - hash - | None -> - if Map.isEmpty h then - "" // Hash file not yet populated (CI before reload-packages) - else - // A non-empty file missing this ref is stale: a ref was added, or an older - // binary regenerated it in place (`growIfNeeded` rewrites it). - Exception.raiseInternal - $"PackageRefs: {kind} hash not found. The hash file is stale; regenerate it with `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages`" - [ "fqn", fqn ] + // The store's answer is NOT cached by generation: the generation tracks the hash file, and a + // rebinding moves the store without touching it. Cheap enough -- one indexed lookup, against + // the interpolated key and Map walk the pinned path pays anyway. + let fromStore = + if kind = "fn" && byNameEnabled.Force() then + resolveFnByName modules name + else + None + + match fromStore with + | Some hash -> + record hash + hash + | None -> + + let gen = currentGeneration () + if gen = cachedGen then + cached + else + let fqn = $"""{kind}/{String.concat "." modules}.{name}""" + let h = getHashes () + match Map.tryFind fqn h with + | Some hash -> + record hash + cachedGen <- gen + cached <- hash + hash + | None -> + if Map.isEmpty h then + "" // Hash file not yet populated (CI before reload-packages) + else + // A non-empty file missing this ref is stale: a ref was added, or an older + // binary regenerated it in place (`growIfNeeded` rewrites it). + Exception.raiseInternal + $"PackageRefs: {kind} hash not found. The hash file is stale; regenerate it with `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages`" + [ "fqn", fqn ] module Type = From 8caffec291304b1f6a68b0c61464b65dc72450aa Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 11:03:33 -0400 Subject: [PATCH 08/32] anyone who can write can rebind: reservedOwners goes, and the server folds reservedOwners refused every write under Darklang that was not trusted seeding, and effective=0 kept the server from folding what it was pushed. both were the same defence -- a name binds last-writer-wins across the store, so anything that could write could rebind the names the kernel resolves through, including the router a server serves. both were also the reason the store could not become the source: the only sanctioned writer to Darklang.* was the reload from packages/ that this arc deletes. given up deliberately. the replacement is an op's authority checked at fold time, and it does not exist yet, so for now write access is rebind access. the comments say so where the checks used to be. effective was carrying two meanings and only one was given up. a pushed op is folded now, but it is still not this store's draft -- op_owners says who pushed it, and that is what keeps discard from deleting a peer's work. two tests covered exactly that and both now assert the split rather than the flag. the 17 entry points resolve from the store by default. safe because a candidate is checked against the pinned version's SIGNATURE and a mismatch falls back, loudly, so a wrong edit degrades rather than bricks the cli. proven both ways: compatible rebind -> GOOD:2 the store's printer ran wrong return type -> warning + 2 the built-in one ran --- .../Builtins.Matter/Libs/PM/PackageOps.fs | 11 ++- backend/src/LibDB/Inserts.fs | 83 ++++++++++--------- backend/src/LibDB/PackageManager.fs | 72 +++++++++++++--- backend/src/LibDB/Queries.fs | 10 ++- backend/src/LibExecution/PackageRefs.fs | 24 ++++-- backend/tests/Tests/MultiInstance.Tests.fs | 8 +- backend/tests/Tests/OpTransport.Tests.fs | 19 +++-- packages/darklang/sync/wire.dark | 13 +-- 8 files changed, 165 insertions(+), 75 deletions(-) diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs index b7a2ebf0bd..f72a0d8dc7 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs @@ -500,8 +500,12 @@ let fns (pm : PT.PackageManager) : List = deprecated = NotDeprecated } - // RELAY store: bulk-insert ops + record ownership (owner) in one transaction, NO fold - // (a relay serves blobs, not projections). The perf path for a relay recording pushes. + // SERVER store: bulk-insert ops + record ownership in one transaction, THEN fold. + // + // It folded nothing until 2026-09-12, on the grounds that a server serves blobs rather than + // projections. That also meant it could not see what it hosted: `/m` showed "Nothing here" + // for packages every client had, a seed could not be cut from the hosted set, and pushing new + // code to a server could never change what it ran. All three are wanted, so it folds. { name = fn "scmStoreOps" 0 typeParams = [] parameters = @@ -524,6 +528,9 @@ let fns (pm : PT.PackageManager) : List = uply { try let! n = LibDB.Inserts.storeOpsWithOwner owner (opRecords records) + // Fold what just arrived, so the projection a seed and `/m` read is current. Cheap: + // ~116us an op, and a push is tens of ops. + let! _ = LibDB.Seed.applyUnappliedOps () return resultOk (Dval.int (bigint n)) with ex -> return resultError (Dval.string ex.Message) diff --git a/backend/src/LibDB/Inserts.fs b/backend/src/LibDB/Inserts.fs index d65e67bf98..ac5f611283 100644 --- a/backend/src/LibDB/Inserts.fs +++ b/backend/src/LibDB/Inserts.fs @@ -285,28 +285,18 @@ let rec insertAndApplyOpsWith /// op the store already runs counts 0. Insert with applied=false, fold, then mark applied=true, so a /// mid-fold failure leaves the ops identifiable and retryable. Commit-free: no commit_hash, so every /// op is live. -/// The `owner` field is the first part of a package name, such as -/// `Darklang.Stdlib.List.map`. Names beginning with `Darklang` are treated as -/// bundled first-party code, so only trusted seeding may create those bindings; -/// guest and sync writes reject them. -let reservedOwners : Set = Set.ofList [ "Darklang" ] - -/// The first operation that binds OR unbinds a name under a protected owner. -/// `None` means no protected location is touched. (The old model also had to -/// chase renames unlisting other bindings of a shared hash; that heuristic is -/// gone -- a `SetName` changes exactly its own location, and retiring a name is -/// an explicit `Unbind` -- so the location arms here are the whole surface.) -let reservedOwnerViolation (ops : List) : Option = - let ownersBound (op : PT.PackageOp) : List = - match op with - | PT.PackageOp.SetName(loc, _, _) -> [ loc.owner ] - | PT.PackageOp.Unbind(loc, _) -> [ loc.owner ] - | _ -> [] - ops - |> List.collect ownersBound - |> List.tryFind (fun owner -> Set.contains owner reservedOwners) - |> Option.map (fun owner -> - $"cannot bind a package name under the reserved owner \"{owner}\"; it is reserved for the bundled standard library") +// No reserved owners. `Darklang` used to be refused here for every write that was not trusted +// seeding, which meant guest `run` and arriving sync ops could not bind under it. +// +// Removed deliberately, 2026-09-12: the store is becoming the source, and the only sanctioned +// writer to `Darklang.*` was the reload from `packages/` that the bootstrapping arc deletes. So +// the protection had to go or the standard library would become uneditable. +// +// What it was defending, for whoever puts a security model back: a name binds last-writer-wins +// across the whole store, so anything that could write could rebind anything -- including the +// names the kernel itself resolves through, and including `Darklang.Matter.router`, which is +// what a server serves. The replacement is an op's AUTHORITY, checked at fold time, and it does +// not exist yet. Until it does, anyone who can write can rebind anything. /// Detect a parser placeholder instead of a real content hash. Placeholders /// are empty or contain the package location; real hashes contain only hex @@ -354,16 +344,17 @@ let insertAndApplyOpsAsWip (ops : List) : Task = /// `effective = 1` is the same clause `Queries.getWipOps` carries and for the same reason: ops a client /// pushed to this store are inert, untagged and uncommitted, so without it a discard here deletes data /// this store is only holding for someone else. -/// Safely insert package operations submitted by RUNNING Dark code -- a guest -/// `run`, or ops that arrived over sync. Rejects protected `Darklang` bindings -/// and unstabilized hashes before insertion; trusted seeding does not come -/// through here. +/// Insert package operations submitted by RUNNING Dark code -- a guest `run`, or ops that arrived +/// over sync. Trusted seeding does not come through here. +/// +/// The only check left is the placeholder-hash one, which is a correctness check rather than a +/// permission: a parser that had no store to ask emits a location where a content hash belongs, +/// and storing that would bind a name to something that does not exist. let insertUntrustedOps (ops : List) : Task> = task { - match reservedOwnerViolation ops, placeholderHashViolation ops with - | Some reason, _ - | None, Some reason -> return Error reason - | None, None -> + match placeholderHashViolation ops with + | Some reason -> return Error reason + | None -> let! count = insertAndApplyOpsAsWip ops return Ok count } @@ -373,11 +364,13 @@ let draftDeletes : List = AND op_id IN (SELECT id FROM package_ops WHERE effective = 1 AND commit_hash IS NULL - AND id NOT IN (SELECT op_id FROM op_branches))" + AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners))" "DELETE FROM package_ops WHERE effective = 1 AND commit_hash IS NULL - AND id NOT IN (SELECT op_id FROM op_branches)" ] + AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners)" ] /// Every main op and what it wrote, EXCEPT the ids in `keep`: the ops this build cannot decode, which /// the caller has read by id. Deleting those would delete a peer's committed op for good because this @@ -402,8 +395,8 @@ let wholeMainDeletes (keep : Set) : List = // // Main only. A branch's rows are keyed by its own id, and no main rewrite may touch them. $"DELETE FROM propagation_policy WHERE branch_id = '{PT.BranchId.Main}'" - // `effective = 1`: excludes client-pushed inert ops; see `draftDeletes`. - $"DELETE FROM package_ops WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches){keepUnreadable}" ] + // Hosted ops excluded via `op_owners`; see `draftDeletes` for why. + $"DELETE FROM package_ops WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches) AND id NOT IN (SELECT op_id FROM op_owners){keepUnreadable}" ] /// Main's op ids this build cannot decode. What `wholeMainDeletes` keeps. let unreadableMainOpIds () : Task> = @@ -604,15 +597,23 @@ let storeOpsWithOwner "op_blob", Sql.bytes blob "origin_ts", Sql.string ts ]) - // `effective = 0`: in the log, NEVER folded into this store's own main. Queued-for-folding - // is not enough, since `growIfNeeded` folds everything `applied = 0 AND effective = 1` on - // the next startup. A client pushes its whole log, package tree included, and names bind - // last-writer-wins over the whole store -- `Darklang.Matter.router` among them -- so anyone - // who could write to a relay could change what that relay itself runs. Hosted ops are DATA: - // the relay serves the blobs back verbatim and its own code stays what its binary seeded. + // `effective = 1`: pushed ops ARE folded into this store's main, like any other arriving + // op. The fold happens in the builtin that calls this, and `growIfNeeded` finishes any + // that were interrupted. + // + // This used to be `effective = 0`, so hosted ops were inert data the server served back + // verbatim and never ran. That was a privilege-escalation defence: a name binds + // last-writer-wins across the store, `Darklang.Matter.router` included, so anyone who + // could push could change what the server itself runs. + // + // Given up deliberately, 2026-09-12, with `reservedOwners`. A server that cannot fold + // cannot serve a seed of what it hosts, cannot show it in `/m`, and cannot be deployed to + // by pushing -- and all three are wanted. The replacement is an op's AUTHORITY checked at + // fold time, and it does not exist yet, so for now anyone who can push can rebind + // anything here. let insertOps = "INSERT OR IGNORE INTO package_ops (id, op_blob, applied, effective, origin_ts) - VALUES (@id, @op_blob, 0, 0, @origin_ts)" + VALUES (@id, @op_blob, 0, 1, @origin_ts)" let statements = if owner = "" then diff --git a/backend/src/LibDB/PackageManager.fs b/backend/src/LibDB/PackageManager.fs index 398dd00b9d..6f55fbd4f7 100644 --- a/backend/src/LibDB/PackageManager.fs +++ b/backend/src/LibDB/PackageManager.fs @@ -643,7 +643,26 @@ let mutable private currentBranchIdOpt : Option = None /// branch's parser the moment you stood on one. Returns `None` for anything not bound here, and /// `PackageRefs` falls back to the pinned hash, so a store that predates a ref behaves as before. /// -/// Off unless `DARK_REFS_BY_NAME=1`; see the note on `PackageRefs.resolveFnByName`. +/// Whether two versions of a fn are interchangeable to an F# caller. +/// +/// The kernel calls these seventeen and never takes one apart, so what has to hold is the +/// SIGNATURE: parameter types in order, the return type, and how many type parameters. Names, +/// descriptions and the body are the author's business. +let private sameSignature + (expected : PT.PackageFn.PackageFn) + (candidate : PT.PackageFn.PackageFn) + : bool = + let paramTypes (fn : PT.PackageFn.PackageFn) = + fn.parameters |> NEList.toList |> List.map (fun p -> p.typ) + + paramTypes expected = paramTypes candidate + && expected.returnType = candidate.returnType + && List.length expected.typeParams = List.length candidate.typeParams + +/// The net. Without it a bad rebind of `Cli.executeCliCommand` is a CLI that cannot start, and +/// the only way back is an environment variable you have to know exists. With it, a rebind whose +/// signature does not match what this build compiles against is refused, loudly, and the pinned +/// version is used: a wrong edit degrades rather than bricks. let private resolveKernelFnByName (modules : string list) (name : string) @@ -651,15 +670,48 @@ let private resolveKernelFnByName try let modulesStr = String.concat "." modules - Sql.query - "SELECT item_hash - FROM locations - WHERE owner = 'Darklang' AND modules = @modules AND name = @name - AND item_type = 'fn' AND unlisted_at IS NULL AND source != 'unbind' - LIMIT 1" - |> Sql.parameters [ "modules", Sql.string modulesStr; "name", Sql.string name ] - |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") - |> fun t -> t.Result + let candidateHash = + Sql.query + "SELECT item_hash + FROM locations + WHERE owner = 'Darklang' AND modules = @modules AND name = @name + AND item_type = 'fn' AND unlisted_at IS NULL AND source != 'unbind' + LIMIT 1" + |> Sql.parameters [ "modules", Sql.string modulesStr; "name", Sql.string name ] + |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") + |> fun t -> t.Result + + match candidateHash with + | None -> None + | Some candidate -> + match LibExecution.PackageRefs.pinnedFnHash modules name with + // Nothing pinned to compare against, or the store already agrees with the pin. The second + // is the overwhelmingly common case and costs one string compare. + | None -> Some candidate + | Some pinned when pinned = candidate -> Some candidate + | Some pinned -> + let fnFor (h : string) = (PMPT.Fn.get (PT.Hash h) |> Ply.toTask).Result + + match fnFor pinned, fnFor candidate with + | Some expected, Some actual when sameSignature expected actual -> + Some candidate + | Some _, Some _ -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the store binds Darklang.{modulesStr}.{name} to a version whose signature " + + "is not the one this build expects, so the built-in version is being used instead." + ) + None + | _, None -> + // Bound to content this store does not hold: nothing to call. + LibExecution.PackageRefs.sayOnce ( + $"warning: Darklang.{modulesStr}.{name} is bound to content this store does not have, " + + "so the built-in version is being used instead." + ) + None + | None, _ -> + // The PINNED version is missing from the store. Nothing left to compare against, and + // refusing would leave no version at all, so trust the store. + Some candidate with _ -> // No store yet, or one without the table: migrations run before any of this exists, and a // ref resolved during them must fall back rather than fail the boot. diff --git a/backend/src/LibDB/Queries.fs b/backend/src/LibDB/Queries.fs index f8d4eba666..5d2f77ec4f 100644 --- a/backend/src/LibDB/Queries.fs +++ b/backend/src/LibDB/Queries.fs @@ -398,10 +398,15 @@ let getDraftOps () : Task> = """ SELECT id, op_blob FROM package_ops - -- effective = 1: excludes client-pushed inert ops; see Inserts.draftDeletes. + -- NOT a hosted op: `op_owners` records who pushed each op TO this store, so a row there is + -- somebody else's work this store is only holding. It is folded like any other op now + -- (a server has to be able to see and serve what it hosts), which is exactly why it has + -- to be excluded HERE: `effective` used to carry that distinction, and a discard that + -- counted a peer's push as this store's draft would delete their data. WHERE effective = 1 AND commit_hash IS NULL AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners) ORDER BY created_at ASC, rowid ASC """ |> Sql.executeAsync (fun read -> @@ -471,9 +476,10 @@ let getWipOps () : Task> = -- Excluding them keeps main authoring's WIP-refresh from sweeping a branch's ops into -- main (re-inserting them effective=1 + folding). Branch isolation. -- - -- effective = 1: excludes client-pushed inert ops; see Inserts.draftDeletes. + -- Hosted ops are excluded for the reason `getDraftOps` above spells out. WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners) -- rowid breaks ties: created_at is second-resolution and a batch shares it, and the pairing -- downstream (HashStabilization) is by adjacency. ORDER BY created_at ASC, rowid ASC diff --git a/backend/src/LibExecution/PackageRefs.fs b/backend/src/LibExecution/PackageRefs.fs index 68552b8eb9..fba40797aa 100644 --- a/backend/src/LibExecution/PackageRefs.fs +++ b/backend/src/LibExecution/PackageRefs.fs @@ -137,6 +137,17 @@ let setHashes (hashes : Map) : unit = hashGeneration <- hashGeneration + 1 +/// Say something once, however many refs trip over it. Public because the resolver that needs it +/// lives in `LibDB`, on the other side of the dependency. +let sayOnce (msg : string) : unit = warn msg + +/// The hash this BUILD pins for a fn ref, which is what "the signature this build expects" means: +/// the pinned version is in the store too, because content is never deleted, so a candidate can be +/// compared against it without pinning anything further. +let pinnedFnHash (modules : string list) (name : string) : string option = + let fqn = $"""fn/{String.concat "." modules}.{name}""" + getHashes () |> Map.tryFind fqn + /// How a FN ref resolves against the live store, when it does. /// /// Installed by whoever owns the store, because `LibDB` depends on `LibExecution` and not the @@ -152,12 +163,15 @@ let setHashes (hashes : Map) : unit = let mutable resolveFnByName : (string list -> string -> string option) = fun _ _ -> None -/// Off by default. By-name resolution means the binary runs whatever the store currently binds -/// `Cli.executeCliCommand` to, which is the intent -- and also means a store with a broken entry -/// point produces a CLI that cannot start. Opt in with `DARK_REFS_BY_NAME=1` until that trade has -/// been made deliberately. +/// ON by default: the store is the source, and these seventeen are the places that matters most. +/// Edit the pretty-printer and the binary you already have starts using it. +/// +/// Safe to default because the resolver checks the candidate's SIGNATURE against the pinned +/// version and refuses a mismatch, loudly, falling back to the pin -- so a wrong edit degrades +/// rather than bricks the CLI. `DARK_REFS_BY_NAME=0` turns it off anyway, which is the escape +/// hatch for a store broken in some way the signature check does not catch. let private byNameEnabled : Lazy = - lazy (System.Environment.GetEnvironmentVariable "DARK_REFS_BY_NAME" = "1") + lazy (System.Environment.GetEnvironmentVariable "DARK_REFS_BY_NAME" <> "0") /// Shared body of `Type.p` and `Fn.p`: a closure resolving `/.` /// against the hash file. Resolution is cached once per hash generation: the answer diff --git a/backend/tests/Tests/MultiInstance.Tests.fs b/backend/tests/Tests/MultiInstance.Tests.fs index 14fe3fbc1a..6b2826bf95 100644 --- a/backend/tests/Tests/MultiInstance.Tests.fs +++ b/backend/tests/Tests/MultiInstance.Tests.fs @@ -691,14 +691,14 @@ let hostedOpsAreNotThisStoresDraft = [ "id", Sql.string (string hostedId) ] Expect.equal stillThere 1L "and a discard leaves it where it is" + // Folded, not inert. A server that cannot fold cannot see or serve what it hosts, so it + // folds now -- and `op_owners`, not `effective`, is what keeps a peer's push out of this + // store's draft, which is the assertion above. let! effective = Sql.query "SELECT effective AS e FROM package_ops WHERE id = @id" |> Sql.parameters [ "id", Sql.string (string hostedId) ] |> Sql.executeRowAsync (fun read -> read.int64 "e") - Expect.equal - effective - 0L - "still inert: a relay serves what it is handed, it does not run it" + Expect.equal effective 1L "and it is folded like any other arriving op" }) diff --git a/backend/tests/Tests/OpTransport.Tests.fs b/backend/tests/Tests/OpTransport.Tests.fs index 016ec8b9ab..0b6a6ef9b1 100644 --- a/backend/tests/Tests/OpTransport.Tests.fs +++ b/backend/tests/Tests/OpTransport.Tests.fs @@ -114,21 +114,28 @@ let foldQuarantinesPoison = /// /// This asserts the flag rather than the consequence because the consequence needs a second process. let relayStoreDoesNotQueueForFolding = - testTask "storeOpsWithOwner: hosted ops are never queued for this store's fold" { + testTask "storeOpsWithOwner: a hosted op is folded, but is not this store's draft" { do! cleanup () let id = "fada0000-0000-0000-0000-0000000000ef" let! n = Inserts.storeOpsWithOwner "someone" [ (id, "00", ts) ] Expect.equal n 1L "stored the op" + // It used to be stored `effective = 0` so no fold would ever touch it. A server has to be + // able to see and serve what it hosts, so it is folded now like any other arriving op. let! effective = effectiveOf id - Expect.equal - effective - 0L - "hosted ops are stored effective=0, so growIfNeeded never folds them into this store's main" + Expect.equal effective 1L "queued for the fold" let! applied = appliedOf id - Expect.equal applied 0L "and they are not pretended to have been applied" + Expect.equal applied 0L "but not pretended to have been applied already" + + // The half that must NOT change. `effective` was carrying two meanings -- "fold this" and + // "this is mine" -- and only the first was given up. An op a peer pushed here is their work, + // so it is not in this store's draft and a discard must leave it alone. + let! draft = LibDB.Queries.getDraftOps () + let inDraft = + draft |> List.exists (fun op -> string (Inserts.computeOpHash op) = id) + Expect.isFalse inDraft "a hosted op is not this store's draft" do! cleanup () } diff --git a/packages/darklang/sync/wire.dark b/packages/darklang/sync/wire.dark index d909b80cfe..eb6caafceb 100644 --- a/packages/darklang/sync/wire.dark +++ b/packages/darklang/sync/wire.dark @@ -201,9 +201,13 @@ let localMaxRowid () : Int64 = |> Stdlib.Option.withDefault 0L -/// RELAY side of a push: store the pushed ops AND record who pushed them, so the owner can pull their -/// stuff back from any machine. Bulk (ops + ownership in one transaction) and store-only -- NO fold, since -/// a relay serves op blobs, not projections. Returns the count of newly-stored ops. +/// SERVER side of a push: store the pushed ops, record who pushed them so the owner can pull their +/// stuff back from any machine, and FOLD them into main. Returns the count of newly-stored ops. +/// +/// It stored without folding until 2026-09-12, which is why the package browser said "Nothing here" +/// for code every client had: the server's own projection never saw what it hosted. Folding is what +/// lets a seed be cut from the hosted set, and what makes pushing new code to a server able to change +/// what it runs. let storeWithOwner (owner: String) (ops: List) @@ -213,8 +217,7 @@ let storeWithOwner match Builtin.scmStoreOps owner records with | Ok n -> - // The author's commits, kept beside the hosted ops so a pull hands them on. Stored, never folded, - // like the ops themselves: a hosted op is inert here whatever commit it names. + // The author's commits, kept beside the hosted ops so a pull hands them on. PackageOps.adoptCommits commits (ops |> Stdlib.List.map (fun op -> (op.id, op.commit))) Stdlib.Result.Result.Ok n | Error e -> Stdlib.Result.Result.Error e From 1bf3c08d54d73947004d93a8692fba8ecac50ab9 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 11:07:56 -0400 Subject: [PATCH 09/32] gate: a push reaches the server's projection, and stays out of its draft crosses a process and a machine boundary -- a real server on a port, a real client with its own store, a real push over http -- because the whole question is whether the SERVER's projection moved, and neither half can be faked in-process. it asserts both halves of what effective=0 used to mean: the op is folded, so the browser can see a package the server hosts; and it is NOT the server's draft, so a discard leaves what its clients pushed alone. the second is the one that would be data loss, and nothing else covers it end to end. in the CI subset. also drops a comment in relay-routes that said the relay never folds what it hosts. --- scripts/testing/_gates-sync | 103 +++++++++++++++++++++++++++++++++++- scripts/testing/gates | 4 +- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/scripts/testing/_gates-sync b/scripts/testing/_gates-sync index 07a75b44fa..c0fc3988e6 100644 --- a/scripts/testing/_gates-sync +++ b/scripts/testing/_gates-sync @@ -97,6 +97,105 @@ gate_setup() { # # Gate, not an F#/.dark test: a real relay on a real port, answering real requests. An HTTP # status and a leaked error body are properties of a request, not of a function call. +# --- the server folds what it is pushed -------------------------------------------------------- +# +# Crosses a PROCESS boundary and a machine boundary: a real server on a port, a real client with +# its own store, a real push over HTTP. Neither half can be faked in-process, because the whole +# question is whether the SERVER's projection moved. +# +# It did not, until 2026-09-12. A pushed op was stored `effective = 0` and never folded, so the +# server could not see what it hosted: the package browser said "Nothing here" for code every +# client had, and no seed could be cut from the hosted set. +# +# The half that must NOT come back with it: a pushed op is folded but is NOT this store's draft. +# `effective` was carrying both meanings and only the first was given up, so `op_owners` carries +# the second. A discard that counted a peer's push as local work would delete their data. +gate_server_folds() { + set -euo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + WORK=rundir/server-folds-test + PORT=${PORT:-9097} + SECRET='server-folds-secret' + + rm -rf "$WORK" + mkdir -p "$WORK/server/logs" "$WORK/client/logs" + scripts/testing/_copy-store "$WORK/server/data.db" + scripts/testing/_copy-store "$WORK/client/data.db" + + cli() { env DARK_CONFIG_RUNDIR="$PWD/$WORK/client/" HOME="$PWD/$WORK/client" "$PWD/$CLI" "$@"; } + + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" relay_grants "$PWD/$CLI" "$PORT" + cli permissions allow package-write > /dev/null 2>&1 + + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + > "$WORK/server/serve.log" 2>&1 & + SRV=$! + trap 'kill $SRV 2>/dev/null' EXIT + + up=false + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && { up=true; break; } + sleep 1 + done + [[ "$up" == "true" ]] || { + echo "the server never came up on $PORT (see $WORK/server/serve.log)" >&2 + exit 2 + } + + failures=0 + fail() { echo " FAIL $1" >&2; failures=$((failures + 1)); } + + name="Tests.ServerFolds.pushed" + cli fn "$name" '() : String = "pushedtoaserver"' > /dev/null 2>&1 + cli commit "server-folds gate" -y > /dev/null 2>&1 + + before=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'pushed' AND modules LIKE '%ServerFolds%';") + [[ "$before" == "0" ]] || fail "the server already knew the name before the push ($before rows)" + + cli connect "http://localhost:$PORT" --secret "$SECRET" > /dev/null 2>&1 + cli push "http://localhost:$PORT" > /dev/null 2>&1 + + after=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'pushed' AND modules LIKE '%ServerFolds%';") + if [[ "$after" == "1" ]]; then + echo " ok a pushed op reaches the server's own projection" + else + fail "the server stored the push but did not fold it ($after location rows)" + fi + + if curl -s --max-time 5 "http://localhost:$PORT/search?q=ServerFolds" | grep -q "ServerFolds"; then + echo " ok and the package browser can see it" + else + fail "the browser still cannot see a package the server hosts" + fi + + # Folded, but not mine. Hosted ops carry an `op_owners` row, and every draft clause excludes + # them; without that a discard on the server deletes whatever its clients pushed. + hosted=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_ops p + WHERE p.id IN (SELECT op_id FROM op_owners) + AND p.effective = 1;") + [[ "$hosted" -gt 0 ]] || fail "no hosted op is effective, so nothing was folded at all" + + draftable=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_ops + WHERE effective = 1 AND commit_hash IS NULL + AND id NOT IN (SELECT op_id FROM op_branches) + AND id IN (SELECT op_id FROM op_owners);") + if [[ "$draftable" == "0" ]]; then + echo " ok and a hosted op is not the server's own draft, so a discard leaves it alone" + else + fail "$draftable hosted op(s) look like this store's draft; a discard would delete them" + fi + + [[ "$failures" == "0" ]] || exit 1 + echo "server-folds: a push reaches the server's projection, and stays out of its draft." +} + gate_relay_routes() { set -euo pipefail @@ -113,9 +212,9 @@ gate_relay_routes() { # the first space. SECRET='relay-test-Bearer secret' - # What a relay needs, under the effect/permission system: the port it binds, the env var its + # What a server needs, under the effect/permission system: the port it binds, the env var its # write secret arrives in, and the store it exists to hold. Granted per instance, and a gate is - # its own instance. Nothing here widens what the relay DOES: it still never folds what it hosts. + # its own instance. DARK_CONFIG_RUNDIR="$PWD/$WORK" relay_grants "$PWD/$CLI" "$PORT" DARK_CONFIG_RUNDIR="$PWD/$WORK" DARK_MATTER_WRITE_SECRET="$SECRET" \ diff --git a/scripts/testing/gates b/scripts/testing/gates index 12e20e1c50..e10f79ef92 100755 --- a/scripts/testing/gates +++ b/scripts/testing/gates @@ -23,6 +23,7 @@ set -uo pipefail GATES=( setup relay-routes + server-folds sync-hostile-relay sync-multi-instance lsp-branches @@ -37,13 +38,14 @@ GATES=( # What CI runs, in one config line: the three gates it has always run, the ones that need only # the debug build already in backend/Build. first-day wants a published artifact and the sync # gates add minutes; those run locally, by name or via `all`. -CI_GATES=(relay-routes workbench-scm workbench-views) +CI_GATES=(relay-routes server-folds workbench-scm workbench-views) describe() { # shellcheck disable=SC2016 # the backticks are prose, not expansion case "$1" in setup) echo '`dark sync setup`, end to end, own rundir' ;; relay-routes) echo "the relay's HTTP surface, given bad input" ;; + server-folds) echo "a push reaches the server's projection, and stays out of its draft" ;; sync-hostile-relay) echo 'the sync/branch CLIENTS, given a relay that lies' ;; sync-multi-instance) echo 'four instances, branches, review queues, agents' ;; lsp-branches) echo 'the `dark/*` branch surface, as an editor drives it' ;; From 1b680870b15df420c272326502148fb25b474e0f Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 11:16:01 -0400 Subject: [PATCH 10/32] dark revert: put a name back to what it held at a commit a revert is a rebinding, not a recovery. every version is still in the store because content is content-addressed and never deleted, so pointing the name at an old hash is the whole operation -- which is why it is symmetric, and why dark log still lists the version you moved off. "at that commit" means that commit or any ancestor, walked through commits.parent. you name a commit to mean a point in time, and the version you want was usually bound by something earlier. that ancestry is also what made this buildable: i had parked revert as needing commit ordering that did not exist, and the column was there. short hashes resolve like git's, and an ambiguous prefix is refused rather than guessed: two commits sharing a prefix is rare, and picking one would revert to the wrong history. the two store reads live in the SCM silo, marked main-scoped: a past binding is a fact about what main committed, and a branch has committed nothing. --- backend/tests/Tests/CliPackages.Tests.fs | 67 ++++++++ packages/darklang/cli/packages/revert.dark | 147 ++++++++++++++++++ packages/darklang/cli/registry.dark | 3 +- packages/darklang/scm/packageOpsBindings.dark | 57 +++++++ 4 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 packages/darklang/cli/packages/revert.dark diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index e6ffc8587e..fc74ab9b1d 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -822,6 +822,71 @@ let aVersionMovedAndMovedBackKeepsTheLastNaming = /// `grep` searches BODIES, which is the half `search` does not do. Scoped to one module on /// purpose: unscoped it renders every live item, and the point of the scope argument is that a /// test, like a person, usually knows roughly where to look. +/// A revert is a rebinding, not a recovery: every version is still in the store, so pointing the +/// name back at an old hash is the whole operation. Which is why it is symmetric. +let revertPutsANameBackAndIsSymmetric = + instanceTest + "revert restores the version a commit held, and reverting twice is a no-op" + (fun state -> + task { + do! start state + do! fn state "Tests.Rv.f" "() : Int64 = 111L" + do! commit state "rv one" + + let! log = runCliPlain state [ "log" ] + // The newest commit is first, and that is the one holding 111. + let first = + log.Split('\n') + |> Array.tryPick (fun line -> + System.Text.RegularExpressions.Regex.Match(line, "[0-9a-f]{8}") + |> fun m -> if m.Success then Some m.Value else None) + + let commitOfOne = + match first with + | Some c -> c + | None -> Tests.failtestf "no commit hash in `dark log`:\n%s" log + + do! fn state "Tests.Rv.f" "() : Int64 = 222L" + do! commit state "rv two" + do! evals state "Tests.Rv.f ()" "222" "the newer version is live" + + do! run state [ "revert"; "Tests.Rv.f"; commitOfOne ] + do! evals state "Tests.Rv.f ()" "111" "and the revert put the old one back" + + do! + shows + state + [ "revert"; "Tests.Rv.f"; commitOfOne ] + "already holds" + "reverting to where it already is says so rather than authoring a no-op" + do! discardAll state + }) + +/// The three ways to get it wrong, which is where a two-argument command earns its errors. +let revertRefusesWhatItCannotFind = + instanceTest + "revert names what it could not find, for the name and for the commit" + (fun state -> + task { + do! start state + do! + shows + state + [ "revert"; "Tests.Rv.nothingHere"; "abc12345" ] + "nothing here is named" + "an unknown name" + do! fn state "Tests.Rv.known" "() : Int64 = 1L" + do! commit state "rv known" + do! + shows + state + [ "revert"; "Tests.Rv.known"; "ffffffffff" ] + "no single commit here starts with" + "an unknown commit" + do! shows state [ "revert" ] "usage: dark revert" "bare prints usage" + do! discardAll state + }) + let grepFindsSourceAndNotJustNames = instanceTest "grep matches a function body, and reports name and line" @@ -1019,6 +1084,8 @@ let tests : List = grepAgreesWithItselfOnceCached grepSaysWhenItFindsNothing grepSeesTheBranchYouAreStandingOn + revertPutsANameBackAndIsSymmetric + revertRefusesWhatItCannotFind aDocOnlyEditKeepsTheVersionAndStillLands aFieldsDocEditLands anEnumCasesDocEditLands diff --git a/packages/darklang/cli/packages/revert.dark b/packages/darklang/cli/packages/revert.dark new file mode 100644 index 0000000000..a886e2ea0e --- /dev/null +++ b/packages/darklang/cli/packages/revert.dark @@ -0,0 +1,147 @@ +module Darklang.Cli.Packages.Revert + +// `dark revert ` -- put a name back to what it held at a past commit. +// +// Nothing is undone and nothing is recovered: every version an item ever had is still in the +// store, because content is content-addressed and never deleted. So a revert is a rebinding -- +// one `SetName`, pointing the name at a hash it used to point at. +// +// Which means it is symmetric. Reverting a revert is just another revert, and `dark log ` +// still shows every version including the one you moved away from. +// +// "At that commit" means the commit or any of its ANCESTORS, not the ops that commit happens to +// name. You give a commit to mean a point in time, and the version you want was usually bound by +// something earlier. + +/// The op a revert authors: bind the name to the old hash, recording what it replaced. +/// +/// `previous = Some current` is the lineage, and it is what stops a peer's merge from reading this +/// as two machines inventing the same name from nothing. +let ops + (location: LanguageTools.ProgramTypes.PackageLocation) + (target: LanguageTools.ProgramTypes.Reference) + (current: LanguageTools.ProgramTypes.Hash) + : List = + [ LanguageTools.ProgramTypes.PackageOp.SetName( + location, + target, + Stdlib.Option.Option.Some current + ) ] + + +/// The same kind of reference the name holds now, pointing at . A revert cannot change +/// what KIND of thing a name holds -- a fn cannot become a type -- so the current binding decides. +let referenceFor + (current: LanguageTools.ProgramTypes.Reference) + (hash: LanguageTools.ProgramTypes.Hash) + : LanguageTools.ProgramTypes.Reference = + match current with + | PackageFn _ -> LanguageTools.ProgramTypes.Reference.PackageFn hash + | PackageType _ -> LanguageTools.ProgramTypes.Reference.PackageType hash + | PackageValue _ -> LanguageTools.ProgramTypes.Reference.PackageValue hash + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + match args with + | [ name, commit ] -> + let here = state.packageData.currentLocation + + match Packages.Location.parseRelativeTo here name with + | Error e -> + Cli.printErr $"the name to revert: {e}" + { state with exitCode = 1 } + + | Ok location -> + let dotted = + PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location + + match Packages.Rename.referenceAt state.currentBranchId location with + | None -> + Cli.printErr $"nothing here is named {dotted}." + Cli.printHint " `dark log` shows what has been committed; `dark ls` what is live." + { state with exitCode = 1 } + + | Some current -> + let currentHash = LanguageTools.ProgramTypes.Reference.hash current + + let currentHashText = + LanguageTools.ProgramTypes.hashToString currentHash + + match SCM.PackageOps.resolveCommit commit with + | None -> + Cli.printErr $"no single commit here starts with {commit}." + Cli.printHint " `dark log` lists them; a longer prefix disambiguates." + { state with exitCode = 1 } + + | Some fullCommit -> + + match SCM.PackageOps.bindingAsOfCommit location fullCommit with + | None -> + Cli.printErr $"{dotted} was not bound at {commit} or anywhere before it." + Cli.printHint $" `dark log {name}` shows the commits that touched it." + + Cli.printHint + " a revert goes back to what the name held THEN, so one added later has nothing to go back to." + + { state with exitCode = 1 } + + | Some oldHash -> + if oldHash == currentHashText then + Cli.printOk + $"{dotted} already holds what it held at {commit} -- nothing to do." + + state + else + let target = + referenceFor current (LanguageTools.ProgramTypes.Hash.Hash oldHash) + + match + SCM.PackageOps.add + state.currentBranchId + (ops location target currentHash) + with + | Ok _ -> + let short = Stdlib.String.slice oldHash 0 8 + Cli.printOk $"reverted {dotted} to the version it held at {commit} ({short})." + + Cli.printHint + " nothing was thrown away: the version you moved off is still in the store, and `dark log` still lists it." + + Cli.printHint " `dark status` shows the change; it travels when you sync." + + state + + | Error e -> + Cli.printErr $"couldn't revert: {e}" + { state with exitCode = 1 } + + | _ -> + Cli.printErr "usage: dark revert " + Cli.printHint " reverts the name to what it held at that commit, or at any commit before it." + { state with exitCode = 1 } + + +let help (_state: Cli.AppState) : String = + [ "" + " dark revert " + "" + " Points a name back at the version it held at that commit. One `SetName`:" + " nothing is undone and nothing is recovered, because every version is still" + " in the store and a name is just a binding." + "" + " \"At that commit\" includes every commit before it, so you can name a point" + " in history rather than the exact commit that happened to rebind the name." + "" + " Symmetric: reverting a revert is another revert, and `dark log` keeps" + " showing every version either way." ] + |> Stdlib.String.join "\n" + + +let complete + (state: Cli.AppState) + (args: List) + : List = + match args with + | [] -> Packages.Nav.complete state args + | [ _ ] -> Packages.Nav.complete state args + | _ -> [] diff --git a/packages/darklang/cli/registry.dark b/packages/darklang/cli/registry.dark index df24b4b4af..da06007ce0 100644 --- a/packages/darklang/cli/registry.dark +++ b/packages/darklang/cli/registry.dark @@ -41,6 +41,7 @@ module Registry = ("module", "Define multiple package declarations", [], Packages.ModuleCommand.execute, Packages.ModuleCommand.help, Packages.ModuleCommand.complete) ("edit", "Change an item without retyping it", [], Packages.Edit.execute, Packages.Edit.help, Packages.Edit.complete) ("rename", "Give an item a different name (its content and callers are untouched)", [], Packages.Rename.execute, Packages.Rename.help, Packages.Rename.complete) + ("revert", "Put a name back to what it held at a past commit", [], Packages.Revert.execute, Packages.Revert.help, Packages.Revert.complete) ("remove", "End a name (the content stays, and its callers keep working)", [], Packages.Remove.execute, Packages.Remove.help, Packages.Remove.complete) ("hash", "Show hash of a package item", [], Packages.Hash.execute, Packages.Hash.help, Packages.Hash.complete) // SCM commands @@ -199,7 +200,7 @@ module Registry = // Command groups organized by category (shared between compact and detailed views) let commandGroups () : List<(String * List)> = - [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "grep", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "hash", "db", "deprecate", "delete", "undeprecate" ]) + [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "grep", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "revert", "hash", "db", "deprecate", "delete", "undeprecate" ]) ("Changes", [ "status", "commit", "commits", "show", "discard", "undo", "propagate", "conflicts", "constraints", "ack", "ops" ]) ("Branches", [ "branch", "branches", "switch", "merge", "rebase", "diff", "log", "resolve" ]) diff --git a/packages/darklang/scm/packageOpsBindings.dark b/packages/darklang/scm/packageOpsBindings.dark index db2d9e78a4..2a2f404596 100644 --- a/packages/darklang/scm/packageOpsBindings.dark +++ b/packages/darklang/scm/packageOpsBindings.dark @@ -233,6 +233,63 @@ let chainBindings (branchId: Uuid) : List = chainBindingEntries branchId |> Stdlib.List.filterMap (fun w -> w.binding) +/// The full hash of the commit names, or None when nothing or more than one does. +/// +/// People type what `dark log` printed, which is the short form. Ambiguity is None rather than a +/// guess: two commits sharing a prefix is rare and picking one silently would revert a name to +/// the wrong history. +/// +/// main-scoped: commits are main's history. A branch's own work is in its overlay, not here. +let resolveCommit (prefix: String) : Stdlib.Option.Option = + let sql = "SELECT hash FROM commits WHERE hash LIKE @p0 || '%' LIMIT 2" + + match Stdlib.Sqlite.queryP (Stdlib.LocalStore.path ()) sql [ prefix ] with + | Ok [ row ] -> Stdlib.Sqlite.textField row "hash" + | _ -> Stdlib.Option.Option.None + + +/// What was bound to AS OF : the newest binding written by any op in +/// that commit or one of its ancestors. `None` when the name was not bound anywhere in that history. +/// +/// Ancestry rather than "ops in that commit": you ask about a commit to mean a point in time, and +/// the name was usually bound by something earlier. `commits.parent` is the chain, and it exists +/// because a commit's id is derived over its parent, the way git's is. +/// +/// History is main's, so the read is main-scoped. A past binding is a fact about what was +/// committed, and a branch has not committed anything -- its own work is in its overlay, which +/// `chainBindingEntries` answers for and this deliberately does not. +let bindingAsOfCommit + (loc: LanguageTools.ProgramTypes.PackageLocation) + (commitHash: String) + : Stdlib.Option.Option = + let modules = Stdlib.String.join loc.modules "." + + // main-scoped: this asks what main's history HELD, including bindings long superseded, so it + // reads unlisted rows too -- which is the one question `locations` is the right table for. + let sql = + "WITH RECURSIVE ancestry(h) AS ( + SELECT @p0 + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent != '' + ) + SELECT l.item_hash + FROM locations l JOIN package_ops p ON p.id = l.op_id + WHERE l.owner = @p1 AND l.modules = @p2 AND l.name = @p3 + AND l.source != 'unbind' + AND p.commit_hash IN (SELECT h FROM ancestry) + ORDER BY l.origin_ts DESC, l.rowid DESC + LIMIT 1" + + match + Stdlib.Sqlite.queryP + (Stdlib.LocalStore.path ()) + sql + [ commitHash, loc.owner, modules, loc.name ] + with + | Ok (row :: _) -> Stdlib.Sqlite.textField row "item_hash" + | _ -> Stdlib.Option.Option.None + + /// EVERY name live on , as bindings. The enumerating form of . /// /// Main's projection, then the chain overlay laid over it: a branch's SetName replaces main's entry From ee2b630bdcccc11009560d32f82f84a10bfde0bf Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 11:57:45 -0400 Subject: [PATCH 11/32] Serve the store as a seed, cut at a commit The package server hands out its op log as a file now: `/seed/latest.db`, `/seed/.db`, and `/seed/meta` for the four fields a pin needs without the twelve megabytes. `latest.db` resolves to the head and is served as the cut AT that head, so it is the same thing as asking for that commit by name. Cuts are cached beside the store keyed by commit, which never needs invalidating: commits are immutable and ops only append. A short prefix resolves like git's; an ambiguous one is refused, because a newest-wins tie would pin a package set nobody chose. Every seed carries a `store_meta` stamp: `format` (the op-blob layout, which is what a migrator keys on), `cut_at`, `kernel`, `at`. `cut_at` names a commit even when no cut was asked for, read back after the cut, so a seed always says what it is a cut of. `Releases.runPending` reads it at open: a store from a NEWER format is refused, one carrying none is stamped. The asymmetry is the point -- a store behind this build is the migrator's job, one ahead is not recoverable by reading harder. Three bugs in `exportAt`, all of them only reachable on a process that cuts more than one seed, which is what a server is: - it checkpointed the source through a READ-ONLY connection and then copied the file. The moment there is actually a WAL to fold in, that fails with a disk I/O error. It goes through `Sqlite.Backup.toFile`, the online-backup API, which also means it exports the store a test repointed LibDB at rather than the config one. - `connStringFor` sets `Pooling=true`, and a pooled connection outlives its `Close`, so the second cut got a handle to a file the first had deleted and replaced. Backup targets are unpooled now. - the seed was left in WAL mode, so the `.db` alone was not the whole database and whether a fetch got everything depended on when a checkpoint ran. It ends in `journal_mode=DELETE`; `LibDB.Sqlite` puts a store back into WAL at open. Folding is not running. `evaluateAllValues` now skips a value whose bindings are all hosted, so a pushed `val` is folded, browsable and servable in a seed, and never executed here -- whoever fetches it evaluates it under their own policy. "All" and not "any": content is shared, so a value this store authored can also arrive by push from a peer who wrote the same thing, and anything locally bound is still evaluated. `op_owners` is empty on an instance, so a client selects exactly what it selected before. Two gates, both crossing a machine boundary because the questions are about what a SERVER did. `seed-serving` tests the pin property by moving the store underneath it: cut at a commit, push, delete the cache so the cut is recomputed, ask again, same ops and no later commit. `server-folds` gained the evaluation half, which needed a real push and then a RESTART, since the startup grow is what would have run the body. --- .../Builtins/Builtins.Matter/Libs/PM/Seed.fs | 14 +- backend/src/LibDB/Releases.fs | 46 ++++ backend/src/LibDB/Seed.fs | 173 +++++++++++-- backend/src/LibDB/Sqlite.fs | 15 +- backend/src/LocalExec/LocalExec.fs | 42 +++- packages/darklang/cli/exportSeed.dark | 2 +- packages/darklang/scm/packageOpsCommits.dark | 42 ++++ packages/darklang/scm/storeMeta.dark | 42 ++++ packages/darklang/stdlib/localStore.dark | 18 ++ packages/darklang/sync/relay/protocol.dark | 182 ++++++++++++++ packages/darklang/sync/relay/server.dark | 39 +-- scripts/testing/_gates-sync | 231 +++++++++++++++++- scripts/testing/gates | 6 +- 13 files changed, 805 insertions(+), 47 deletions(-) create mode 100644 packages/darklang/scm/storeMeta.dark diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs index a82599ad2a..fa01a51dbb 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs @@ -6,6 +6,8 @@ open LibExecution.Effects module Dval = LibExecution.Dval module Builtin = LibExecution.Builtin +module C2DT = LibExecution.CommonToDarkTypes +module D = LibExecution.DvalDecoder open Builtin.Shortcuts @@ -13,19 +15,25 @@ open Builtin.Shortcuts let fns : List = [ { name = fn "pmSeedExport" 0 typeParams = [] - parameters = [ Param.make "outputPath" TString "" ] + parameters = + [ Param.make "outputPath" TString "" + Param.make + "upToCommit" + (TypeReference.option TString) + "cut the seed at this commit and its ancestors, so the same commit yields the same ops however far the store has moved since; `None` takes everything committed" ] returnType = TypeReference.result TUnit TString description = "Export a minimal seed.db from the current database" fn = let resultOk = Dval.resultOk KTUnit KTString let resultError = Dval.resultError KTUnit KTString (function - | state, vm, _, [| DString outputPath |] -> + | state, vm, _, [| DString outputPath; upToCommit |] -> uply { try let outputPath = LibExecution.Host.normalizeFilePath outputPath LibExecution.PermissionCheck.requireFileWrite state vm outputPath - do! LibDB.Seed.export outputPath + let upToCommit = C2DT.Option.fromDT D.string upToCommit + do! LibDB.Seed.exportAt outputPath upToCommit return resultOk DUnit with ex -> return resultError (DString ex.Message) diff --git a/backend/src/LibDB/Releases.fs b/backend/src/LibDB/Releases.fs index b21dc04e2c..8016707b8d 100644 --- a/backend/src/LibDB/Releases.fs +++ b/backend/src/LibDB/Releases.fs @@ -249,6 +249,50 @@ let steps : List = ] +/// The format the store says it was written in, if it says. +/// +/// Absent means a store older than the stamp, which is every store built before seeds carried one. +/// That is not an error: it predates the field, and its ops are format 1 by construction. +let storedFormat () : Option = + if not (tableExists "store_meta") then + None + else + Sql.query "SELECT value FROM store_meta WHERE key = 'format'" + |> Sql.execute (fun read -> read.string "value") + |> Result.unwrap + |> List.tryHead + |> Option.bind (fun v -> + match System.UInt32.TryParse v with + | true, n -> Some n + | false, _ -> None) + + +/// Refuse a store written by a NEWER build's format, and stamp one that carries no format yet. +/// +/// The asymmetry is the point. A store BEHIND this build is the migrator's job (it can read an old +/// layout, because every historical reader stays in the binary). A store AHEAD of it is not +/// recoverable by reading harder: the layout is one this binary has never seen, and guessing at it +/// corrupts the only copy of the ops. +let private checkFormat () : unit = + match storedFormat () with + | Some n when n > LibSerialization.Binary.BaseFormat.CurrentVersion -> + Exception.raiseInternal + "this store was written by a newer Darklang than this one and cannot be read safely" + [ "store format", string n + "this build reads", string LibSerialization.Binary.BaseFormat.CurrentVersion ] + | _ -> + // Stamp it, so from here every store says what it is. `INSERT OR REPLACE` rather than a + // conditional: the value is the same whether the row was missing or already right. + Sql.query + "CREATE TABLE IF NOT EXISTS store_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + |> Sql.executeStatementSync + + Sql.query "INSERT OR REPLACE INTO store_meta (key, value) VALUES ('format', @v)" + |> Sql.parameters + [ "v", Sql.string (string LibSerialization.Binary.BaseFormat.CurrentVersion) ] + |> Sql.executeStatementSync + + let private alreadyRun () : Set = if not (tableExists "system_migrations_v0") then Set.empty @@ -323,6 +367,8 @@ let applySchemaIndexes (schemaSql : string) : unit = /// makes it safe to run against a store of any age, so a new step that skips those has nothing /// checking it. let runPending () : unit = + checkFormat () + // A step's name is its identity in `system_migrations_v0`, so two steps sharing one would run as // one and record as one, silently. Refused here, where every store passes on startup, because no // test constructs this list. diff --git a/backend/src/LibDB/Seed.fs b/backend/src/LibDB/Seed.fs index 3d330221a6..aecbe5db87 100644 --- a/backend/src/LibDB/Seed.fs +++ b/backend/src/LibDB/Seed.fs @@ -42,24 +42,39 @@ let mutable private warnedAboutUnreadableOps = false /// Export a seed database to the given output path: copy the full source DB, then strip everything /// that belongs to the machine that built it rather than to the package set (see the DELETEs below). -let export (outputPath : string) : Task = +/// A seed, optionally cut at a COMMIT rather than at now. +/// +/// `Some commit` keeps the ops that commit or one of its ancestors names, and drops the rest, so +/// two people fetching the same commit get the same bytes however far the source has moved since. +/// That immutability is what makes the seed cacheable and the pin reproducible. +/// +/// Ancestry through `commits.parent`, the same walk `revert` uses: a commit names a point in +/// history, not a set of ops. +let exportAt (outputPath : string) (upToCommit : string option) : Task = task { - let sourcePath = LibConfig.Config.dbPath - if System.IO.File.Exists outputPath then System.IO.File.Delete outputPath - // Checkpoint WAL before copying to ensure all data is in the main file - let sourceConnStr = $"Data Source={sourcePath};Mode=ReadOnly;Cache=Private" - use sourceConn = new SqliteConnection(sourceConnStr) - sourceConn.Open() - use checkpointCmd = sourceConn.CreateCommand() - checkpointCmd.CommandText <- "PRAGMA wal_checkpoint(TRUNCATE);" - checkpointCmd.ExecuteNonQuery() |> ignore - sourceConn.Close() - - System.IO.File.Copy(sourcePath, outputPath) - - let connStr = $"Data Source={outputPath};Mode=ReadWriteCreate;Cache=Private" + // Through SQLite's online-backup API, never a file copy. + // + // The store this runs against is LIVE -- a server cuts a seed while serving, and the process + // holding it has a WAL open -- and `data.db` alone is not the store while recent writes sit in + // `data.db-wal`. The previous version checkpointed the source first and then copied the file, + // which cannot work on a long-lived process: the checkpoint went through a ReadOnly connection, + // so the moment there was actually a WAL to fold in it failed with a disk I/O error. + // + // It also reads `Sqlite.connString` rather than the config path, so a test that repoints LibDB + // at its own store exports THAT store rather than the default one. + match Backup.toFile outputPath with + | Error e -> + Exception.raiseInternal $"could not snapshot the store to cut a seed: {e}" [] + | Ok() -> () + + // `Pooling=False`: a pooled connection outlives its `Close`, so on a process that cuts more than + // one seed the second cut is handed a handle to a file the first one has since deleted and + // replaced, and fails with "attempt to write a readonly database". A cut opens one connection and + // happens rarely; pooling buys it nothing. + let connStr = + $"Data Source={outputPath};Mode=ReadWriteCreate;Cache=Private;Pooling=False" use conn = new SqliteConnection(connStr) conn.Open() @@ -136,14 +151,103 @@ let export (outputPath : string) : Task = """ cleanCmd.ExecuteNonQuery() |> ignore + // Cut at a commit: drop every op the commit's history does not name, and every commit outside + // that history. Runs AFTER the clean above, so it only ever narrows what that already kept. + match upToCommit with + | None -> () + | Some commit -> + use cutCmd = conn.CreateCommand() + cutCmd.CommandText <- + """ + CREATE TEMP TABLE seed_ancestry AS + WITH RECURSIVE ancestry(h) AS ( + SELECT hash FROM commits WHERE hash = $commit + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent <> '' + ) + SELECT h FROM ancestry; + + DELETE FROM package_ops + WHERE commit_hash IS NULL OR commit_hash NOT IN (SELECT h FROM seed_ancestry); + + DELETE FROM commits WHERE hash NOT IN (SELECT h FROM seed_ancestry); + + DROP TABLE seed_ancestry; + """ + cutCmd.Parameters.AddWithValue("$commit", commit) |> ignore + cutCmd.ExecuteNonQuery() |> ignore + + // `cut_at` names a commit even when the caller asked for no cut, so EVERY seed says what it is + // a cut of. Read back after the cut, so it is the tip of what the file actually holds rather + // than the tip of the store it came from. + let cutAt = + use tipCmd = conn.CreateCommand() + tipCmd.CommandText <- + "SELECT hash FROM commits ORDER BY created_at DESC, rowid DESC LIMIT 1" + match tipCmd.ExecuteScalar() with + | null -> "" + | tip -> string tip + + // The stamp every seed and every store carries, so a store can say which cut it came from and + // which build made it. Written here because export is the only thing that knows. + // + // `format` is the op-blob layout version, which is what a migrator keys on: a store two + // formats behind needs two steps, and a store from a NEWER format has to be refused rather + // than misread. + use stampCmd = conn.CreateCommand() + stampCmd.CommandText <- + """ + CREATE TABLE IF NOT EXISTS store_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + DELETE FROM store_meta; + INSERT INTO store_meta (key, value) VALUES + ('format', $format), + ('cut_at', $cutAt), + ('kernel', $kernel), + ('at', $at); + """ + stampCmd.Parameters.AddWithValue( + "$format", + string LibSerialization.Binary.BaseFormat.CurrentVersion + ) + |> ignore + stampCmd.Parameters.AddWithValue("$cutAt", cutAt) |> ignore + stampCmd.Parameters.AddWithValue("$kernel", LibConfig.Config.buildHash) + |> ignore + stampCmd.Parameters.AddWithValue( + "$at", + System.DateTime.UtcNow.ToString( + "o", + System.Globalization.CultureInfo.InvariantCulture + ) + ) + |> ignore + stampCmd.ExecuteNonQuery() |> ignore + use vacuumCmd = conn.CreateCommand() vacuumCmd.CommandText <- "VACUUM;" vacuumCmd.ExecuteNonQuery() |> ignore + // Out of WAL before anyone gets the file. A seed is SHIPPED -- copied, served over HTTP, + // embedded in a binary -- and in WAL mode the `.db` on its own is not the whole database, so + // whether it is complete depends on when a checkpoint happened to run. `journal_mode=DELETE` + // folds the WAL back in and removes it, which makes the one file the whole seed. + // `LibDB.Sqlite` puts a store back into WAL at open, so nothing downstream loses it. + use settleCmd = conn.CreateCommand() + settleCmd.CommandText <- + "PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE;" + settleCmd.ExecuteNonQuery() |> ignore + conn.Close() } +/// A seed of main as it stands now. +let export (outputPath : string) : Task = exportAt outputPath None + + // --------------------- // Grow // --------------------- @@ -496,6 +600,37 @@ module ValueEvaluationError = | None -> e.message | Some(PT.Hash h) -> $"Value {h} ({e.location}): {e.message}" +/// The values this store may EVALUATE: not the ones somebody else pushed here. +/// +/// Folding an op is inert -- it deserializes and writes projection rows, and runs no guest code. +/// Evaluating a `val` is not: it executes the body. On a client that distinction does not arise, +/// because everything in its store is either its own or something it chose to pull. On a SERVER it +/// is the whole difference between holding somebody's code and running it, and `op_owners` already +/// records which ops arrived by push. +/// +/// So: a value with a pushed binding and no local one is folded, browsable and servable in a seed, +/// and never executed here. Whoever fetches it evaluates it on their own machine, under their own +/// policy, which is where that decision belongs. +/// +/// "and no local one" matters. Content is shared, so a value this store authored can also arrive by +/// push from a peer who wrote the same thing; anything locally bound is still evaluated. +/// +/// `op_owners` is empty on an instance, so both EXISTS clauses are false there and this selects +/// exactly what it selected before. +let private evaluableValues = + """ + pv.rt_dval IS NULL + AND NOT ( + EXISTS (SELECT 1 FROM locations hl + WHERE hl.item_hash = pv.hash + AND hl.op_id IN (SELECT op_id FROM op_owners)) + AND NOT EXISTS (SELECT 1 FROM locations ll + WHERE ll.item_hash = pv.hash + AND ll.op_id NOT IN (SELECT op_id FROM op_owners)) + ) + """ + + /// Evaluate all package values that have NULL rt_dval, under `authority`. /// Multi-pass: values may depend on other values, so we retry until convergence. let evaluateAllValues @@ -535,11 +670,11 @@ let evaluateAllValues let! unevaluatedValues = Sql.query - """ + $""" SELECT pv.hash, pv.pt_def, l.owner, l.modules, l.name FROM package_values pv LEFT JOIN locations l ON l.item_hash = pv.hash AND l.unlisted_at IS NULL - WHERE pv.rt_dval IS NULL + WHERE {evaluableValues} """ |> Sql.executeAsync (fun read -> let hash = Hash(read.string "hash") @@ -670,8 +805,10 @@ let growIfNeeded // NULL forever, and a NULL `rt_dval` reads as "value not found". Evaluate whenever any value is // unevaluated so the store self-heals on startup. let! hasUnevaluatedValues = + // The same predicate `evaluateAllValues` selects with, or a server would take this branch on + // every startup for hosted values it is never going to evaluate. Sql.query - "SELECT EXISTS(SELECT 1 FROM package_values WHERE rt_dval IS NULL) AS has_null" + $"SELECT EXISTS(SELECT 1 FROM package_values pv WHERE {evaluableValues}) AS has_null" |> Sql.executeRowAsync (fun read -> read.int64 "has_null") |> Task.map (fun n -> n > 0L) if appliedCount > 0L then diff --git a/backend/src/LibDB/Sqlite.fs b/backend/src/LibDB/Sqlite.fs index 3080c94d63..d1d52e9e1b 100644 --- a/backend/src/LibDB/Sqlite.fs +++ b/backend/src/LibDB/Sqlite.fs @@ -13,6 +13,17 @@ open Prelude let private connStringFor (path : string) : string = $"Data Source={path};Mode=ReadWriteCreate;Cache=Private;Pooling=true" +/// A connection to a one-shot FILE beside the store: a backup being written, a seed being cut, a +/// file being restored from. +/// +/// Unpooled, which is the whole difference. A pooled connection outlives its `Close`, so a process +/// that touches the same path twice -- a server cutting a seed at one commit, then at another, into +/// a file it deleted in between -- is handed a handle to the file that is no longer there and fails +/// with "attempt to write a readonly database" or a disk I/O error. The live store wants pooling; +/// a file opened once does not. +let private fileConnStringFor (path : string) : string = + $"Data Source={path};Mode=ReadWriteCreate;Cache=Private;Pooling=False" + let private defaultConnString = connStringFor LibConfig.Config.dbPath /// The store this process is actually reading and writing, which is `LibConfig.Config.dbPath` except @@ -55,7 +66,7 @@ module Backup = /// Snapshot the live store into `target`, creating it. let toFile (target : string) : Result = - copy connString (connStringFor target) + copy connString (fileConnStringFor target) /// Replace the live store's contents with `source`'s. /// @@ -67,7 +78,7 @@ module Backup = if not (System.IO.File.Exists source) then Error $"no file at {source}" else - copy (connStringFor source) connString + copy (fileConnStringFor source) connString module Sql = diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index 7fafa055a2..774ab194c5 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -109,10 +109,13 @@ module HandleCommand = return Error $"Migration failed: {ex.Message}" } - let exportSeed (outputPath : string) : Ply> = + let exportSeed + (outputPath : string) + (upToCommit : string option) + : Ply> = uply { try - do! LibDB.Seed.export outputPath + do! LibDB.Seed.exportAt outputPath upToCommit let size = System.IO.FileInfo(outputPath).Length / 1024L / 1024L print $"Seed exported to {outputPath} ({size} MB)" return Ok() @@ -120,6 +123,22 @@ module HandleCommand = return Error $"Export failed: {ex.Message}" } + /// Write `package-ref-hashes.txt` from whatever store this rundir has. + /// + /// The kernel's entry points are pinned BY HASH, so a binary needs that file before it can resolve + /// anything. The fill path writes it as a side effect of reloading `packages/`; this is the same + /// step on its own, for a store that arrived as a SEED and has no `packages/` to reload. That is + /// the only thing standing between a fetch-at-pin build and a working binary. + let generateRefs () : Ply> = + uply { + try + do! LibDB.PackageRefsGenerator.generate () + LibExecution.PackageRefs.reloadHashes () + return Ok() + with ex -> + return Error $"Generating package refs failed: {ex.Message}" + } + let listMigrations () : Ply> = uply { try @@ -183,7 +202,21 @@ let main (args : string[]) : int = | [ "export-seed"; outputPath ] -> handleCommand $"Exporting seed to {outputPath}" - (HandleCommand.exportSeed outputPath) + (HandleCommand.exportSeed outputPath None) + + // Cut at a commit, so what a pin fetches is fixed by the commit rather than by when it asked: + // the same commit yields the same OPS however far the store has moved since, and ids are derived + // from op content, so two stores built from it agree. Not byte-identical -- the stamp records + // which build cut it and when -- and nothing needs it to be. + | [ "export-seed"; outputPath; commit ] -> + handleCommand + $"Exporting seed at {commit} to {outputPath}" + (HandleCommand.exportSeed outputPath (Some commit)) + + | [ "refs"; "generate" ] -> + handleCommand + "writing package-ref-hashes.txt from this store" + (HandleCommand.generateRefs ()) | [ "pm-sweep-blobs" ] -> handleCommand @@ -206,7 +239,8 @@ let main (args : string[]) : int = print " reload-packages" print " migrations run" print " migrations list" - print " export-seed " + print " export-seed [commit]" + print " refs generate" print " pm-sweep-blobs" print " bench" print " bench-render" diff --git a/packages/darklang/cli/exportSeed.dark b/packages/darklang/cli/exportSeed.dark index 07258de478..fa2d61783b 100644 --- a/packages/darklang/cli/exportSeed.dark +++ b/packages/darklang/cli/exportSeed.dark @@ -8,7 +8,7 @@ let execute (state: AppState) (args: List): AppState = Stdlib.printLine (help state) state | [outputPath] -> - match Builtin.pmSeedExport outputPath with + match Stdlib.LocalStore.seedTo outputPath (Stdlib.Option.Option.None) with | Ok _ -> Stdlib.printLine $"Seed exported to {outputPath}" state diff --git a/packages/darklang/scm/packageOpsCommits.dark b/packages/darklang/scm/packageOpsCommits.dark index ece0bfdb38..094746f102 100644 --- a/packages/darklang/scm/packageOpsCommits.dark +++ b/packages/darklang/scm/packageOpsCommits.dark @@ -72,6 +72,48 @@ let count () : Int64 = |> Stdlib.Option.withDefault 0L +/// What a seed cut at would carry: (ops, commits). `None` when no commit here +/// has that hash. +/// +/// The same ancestry walk `LibDB.Seed.exportAt` cuts with, and the same exclusion of branch ops, so +/// the numbers match what a fetch actually gets. Exists so a pin can be CHECKED without cutting one: +/// the answer is two counts, and the seed is twelve megabytes. +let seedShapeAt (commitHash: String) : Stdlib.Option.Option<(Int64 * Int64)> = + let sql = + "WITH RECURSIVE ancestry(h) AS ( + SELECT hash FROM commits WHERE hash = @p0 + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent <> '' + ) + SELECT + (SELECT count(*) FROM package_ops + WHERE commit_hash IN (SELECT h FROM ancestry) + AND id NOT IN (SELECT op_id FROM op_branches)) AS ops, + (SELECT count(*) FROM ancestry) AS commits" + + match Stdlib.Sqlite.queryOneP (localDb ()) sql [ commitHash ] with + | None -> Stdlib.Option.Option.None + | Some row -> + match (Stdlib.Sqlite.intField row "ops", Stdlib.Sqlite.intField row "commits") with + // A hash nobody here has gives an empty ancestry, so zero commits. That is "not found", not a + // seed with nothing in it. + | (Some _, Some 0L) -> Stdlib.Option.Option.None + | (Some ops, Some commits) -> Stdlib.Option.Option.Some((ops, commits)) + | _ -> Stdlib.Option.Option.None + + +/// How many ops are COMMITTED on main, which is what a seed cut from here carries. +/// +/// Not : that counts the draft and every branch's delta too, and a seed carries neither. +let committedCountOnMain () : Int64 = + Stdlib.Sqlite.scalarInt + (localDb ()) + "SELECT count(*) AS n FROM package_ops + WHERE commit_hash IS NOT NULL AND id NOT IN (SELECT op_id FROM op_branches)" + "n" + |> Stdlib.Option.withDefault 0L + + /// Which ops a branch can SEE: main's log, plus that branch's own delta ops. Another branch's ops are in /// the same table but invisible from here -- that isolation is the point of the overlay. /// diff --git a/packages/darklang/scm/storeMeta.dark b/packages/darklang/scm/storeMeta.dark new file mode 100644 index 0000000000..d85e06b073 --- /dev/null +++ b/packages/darklang/scm/storeMeta.dark @@ -0,0 +1,42 @@ +module Darklang.SCM.StoreMeta + +// The `store_meta` silo: what a store says about ITSELF, as opposed to about the code in it. +// +// Written by the seed cut (`LibDB.Seed.exportAt`, the only thing that knows what it just made) and +// topped up at open by `LibDB.Releases`, so every store carries one whether it came from a seed or +// grew here. Four keys today: +// +// format the op-blob layout version. A reader that understands a different one must refuse. +// cut_at the commit this store was cut at, so a seed says what it is a cut of. +// kernel the build that cut it, for tracing a bad seed back to a binary. +// at when. +// +// Read-only from Dark: the values are facts about a file, and nothing in package code is in a +// position to know better than the thing that wrote them. + + +/// The value of , or "" when this store carries no such key. +/// +/// "" rather than an Option: a store older than the stamp has none of these, which every caller +/// treats the same way as "not recorded" -- the same choice `Stdlib.LocalStore.configGet` makes. +let get (key: String) : String = + match + Stdlib.Sqlite.queryOneP + (localDb ()) + "SELECT value FROM store_meta WHERE key = @p0" + [ key ] + with + | Some row -> Stdlib.Sqlite.textField row "value" |> Stdlib.Option.withDefault "" + | None -> "" + + +/// Everything this store says about itself, for `dark status` and the seed endpoints. +let all () : List<(String * String)> = + Stdlib.Sqlite.rowsOrEmptyP + (localDb ()) + "SELECT key, value FROM store_meta ORDER BY key" + [] + |> Stdlib.List.filterMap (fun row -> + match (Stdlib.Sqlite.textField row "key", Stdlib.Sqlite.textField row "value") with + | (Some k, Some v) -> Stdlib.Option.Option.Some((k, v)) + | _ -> Stdlib.Option.Option.None) diff --git a/packages/darklang/stdlib/localStore.dark b/packages/darklang/stdlib/localStore.dark index de5fea2d0f..f3e278c03f 100644 --- a/packages/darklang/stdlib/localStore.dark +++ b/packages/darklang/stdlib/localStore.dark @@ -44,3 +44,21 @@ let backupTo (path: String) : Stdlib.Result.Result = /// memory is still the OLD store, though, so the caller should say to restart. let restoreFrom (path: String) : Stdlib.Result.Result = Builtin.localDbRestoreFrom path + + +/// Cut a SEED of this store to : the op log and its commits, with every per-instance +/// table stripped, so a fresh install that folds it agrees on every id. +/// +/// narrows it to that commit and its ancestors, which is what makes a pinned +/// fetch reproducible: the same commit yields the same OPS however far the store has moved since, +/// and ids come from op content, so every store built from it agrees. `None` takes everything +/// committed. +/// +/// The file carries a `store_meta` stamp saying which commit it was cut at, which format its op +/// blobs are in, and which build cut it. So a seed says what it is without anyone having to +/// remember where it came from. +let seedTo + (path: String) + (upToCommit: Stdlib.Option.Option) + : Stdlib.Result.Result = + Builtin.pmSeedExport path upToCommit diff --git a/packages/darklang/sync/relay/protocol.dark b/packages/darklang/sync/relay/protocol.dark index 31ac57c11e..4f96c0e2ba 100644 --- a/packages/darklang/sync/relay/protocol.dark +++ b/packages/darklang/sync/relay/protocol.dark @@ -255,3 +255,185 @@ let branchOwnersHandlerFn (req: HttpRequest) : HttpResponse = match branchAccessAllowed req with | Error why -> Stdlib.Http.responseWithText why 401 | Ok _ -> Stdlib.Http.responseWithText (SCM.PackageOps.relayListAllBranches ()) 200 + + +// --------------------- +// Seeds +// --------------------- +// +// A SEED is the op log as a file: what a fresh install folds to become this store, and what CI fetches +// instead of parsing `packages/` out of text. It is cut from the SAME log `/sync/pull` pages out, so the +// two can never disagree about what this relay holds. +// +// Cut AT A COMMIT, always, even when the caller asked for "latest": that is what makes a pin +// reproducible. Two machines asking for the same commit get the same OPS however far the store has +// moved in between, which is the property `scripts/packages/pin` rests on. The file is not +// byte-identical across servers -- the stamp says which build cut it and when -- and nothing needs +// it to be, since ids are derived from op content rather than from the file. + +val seedRoute = "/seed/:file" + + +/// Where a cut seed is cached: beside the store, named for the commit it was cut at. +/// +/// A cut is decided entirely by its commit -- commits are immutable and ops only ever append -- so a +/// file here never needs invalidating. Beside the store rather than inside it, like the source cache: +/// it is derived, it is per-install, and it has no business travelling inside a seed of its own. +let seedCachePath (commit: String) : String = + let storeDir = Stdlib.Cli.Path.parent (Stdlib.LocalStore.path ()) + Stdlib.Cli.Path.join [ storeDir, "seeds", $"{commit}.db" ] + + +/// The commit main is at here, or "" when nothing has been committed yet. +let headCommit () : String = + match SCM.Commits.recent 1L with + | c :: _ -> c.hash + | [] -> "" + + +/// Cut a seed at , or hand back the one already cut there. +let cutSeedAt (commit: String) : Stdlib.Result.Result = + let path = seedCachePath commit + + if Stdlib.Cli.File.exists path then + Stdlib.Result.Result.Ok path + else + match Stdlib.Cli.Dir.createRecursive (Stdlib.Cli.Path.parent path) with + | Error e -> + Stdlib.Result.Result.Error + $"could not make the seed cache directory: {Stdlib.Cli.Posix.Error.toString e}" + | Ok _ -> + match Stdlib.LocalStore.seedTo path (Stdlib.Option.Option.Some commit) with + | Error e -> Stdlib.Result.Result.Error e + | Ok _ -> Stdlib.Result.Result.Ok path + + +/// What a seed cut from this relay says about itself, so a pin can be checked without downloading +/// tens of megabytes to read four fields. +type SeedMeta = + { /// The commit the seed this describes would be cut at: the current head, or the resolved + /// `?commit=` when one was asked about. "" only when nothing is committed here at all. + head: String + /// The op-blob layout a seed from here is written in. A fetcher that reads a different one must + /// refuse rather than fold it. + format: String + /// The build that would cut it, for tracing a bad seed back to a binary. This server's own + /// build, not whatever its store says: the store's `kernel` records the build that cut the seed + /// it GREW from, which is a different fact and older. + kernel: String + /// What a seed cut at `head` would carry. A client comparing these with its own knows whether + /// a fetch would tell it anything. + ops: Int64 + commits: Int64 } + + +/// `/seed/meta`: the fields a pin needs, without the seed. +/// +/// `?commit=` asks about THAT commit instead of the head, which is how a pin is checked: it +/// answers 404 if the commit does not resolve here, and otherwise the shape the cut would have. +/// Two counts rather than twelve megabytes. +let seedMetaHandlerFn (req: HttpRequest) : HttpResponse = + let asked = + Stdlib.Http.Request.queryParam req "commit" |> Stdlib.Option.withDefault "" + + let resolved = + if asked == "" then + Stdlib.Result.Result.Ok(Stdlib.Option.Option.None) + else + match SCM.PackageOps.resolveCommit asked with + | Some hash -> Stdlib.Result.Result.Ok(Stdlib.Option.Option.Some hash) + | None -> Stdlib.Result.Result.Error $"no single commit here starts with \"{asked}\"" + + match resolved with + | Error why -> Stdlib.Http.responseWithText why 404 + | Ok at -> + let shape = + match at with + | Some hash -> SCM.PackageOps.seedShapeAt hash + | None -> + Stdlib.Option.Option.Some( + (SCM.PackageOps.committedCountOnMain (), + SCM.PackageOps.commitCountOnBranch SCM.Branch.mainBranchId) + ) + + match shape with + | None -> Stdlib.Http.responseWithText $"no commit here named \"{asked}\"" 404 + | Some counts -> + let (ops, commits) = counts + + let meta = + SeedMeta + { head = + match at with + | Some hash -> hash + | None -> headCommit () + format = SCM.StoreMeta.get "format" + kernel = Darklang.Cli.Installation.Helpers.buildHash () + ops = ops + commits = commits } + + Stdlib.Http.responseWithJson (Stdlib.Json.serialize meta) 200 + + +/// `/seed/latest.db` and `/seed/.db`: the store as a file, cut at a commit. +/// +/// `latest.db` resolves to the current head and is served as the cut AT that head, so it is the same +/// bytes as asking for that commit by name. A short commit prefix resolves like git's; an ambiguous +/// one is refused rather than guessed at, because guessing here pins the wrong package set. +let seedHandlerFn (req: HttpRequest) : HttpResponse = + let requested = + Stdlib.HttpServer.getPathParam req seedRoute "file" + |> Stdlib.Option.withDefault "" + + // `.db` is part of the route rather than a query param so a browser, `curl -O` and a CI cache all + // end up with a file named like what it is. + if Stdlib.Bool.not (Stdlib.String.endsWith requested ".db") then + Stdlib.Http.badRequest + "a seed is asked for as `/seed/latest.db` or `/seed/.db`" + else + + let asked = Stdlib.String.dropLast requested 3 + + let resolved = + if asked == "" then + // `/seed/.db` asks for nothing. Refused rather than read as an empty prefix, which matches + // every commit and so resolves to one whenever a store happens to hold exactly one. + Stdlib.Result.Result.Error + "a seed is asked for as `/seed/latest.db` or `/seed/.db`" + else if asked == "latest" then + let head = headCommit () + + if head == "" then + Stdlib.Result.Result.Error + "nothing has been committed here yet, so there is no seed to cut" + else + Stdlib.Result.Result.Ok head + else + // `resolveCommit`, not `SCM.Commits.byHashPrefix`: that one takes the newest on a tie, which is + // the right rule for a listing and the wrong one here. An ambiguous prefix would pin a package + // set nobody chose, and silently. + match SCM.PackageOps.resolveCommit asked with + | Some hash -> Stdlib.Result.Result.Ok hash + | None -> + Stdlib.Result.Result.Error + $"no single commit here starts with \"{asked}\"" + + match resolved with + | Error why -> Stdlib.Http.responseWithText why 404 + | Ok commit -> + match cutSeedAt commit with + | Error why -> Stdlib.Http.serverError why + | Ok path -> + match Stdlib.Cli.File.readBytes path with + | Error e -> + Stdlib.Http.serverError + $"cut the seed but could not read it back: {Stdlib.Cli.Posix.Error.toString e}" + | Ok bytes -> + // The commit is in the bytes too (`store_meta.cut_at`), so this header is a convenience for + // a fetcher that wants it without opening SQLite -- not the only record of it. + Stdlib.Http.responseWithHeaders + bytes + [ ("Content-Type", "application/vnd.sqlite3"), + ("X-Dark-Seed-Commit", commit), + ("Content-Disposition", "attachment; filename=\"seed.db\"") ] + 200 diff --git a/packages/darklang/sync/relay/server.dark b/packages/darklang/sync/relay/server.dark index bf235fc06a..15b02677a3 100644 --- a/packages/darklang/sync/relay/server.dark +++ b/packages/darklang/sync/relay/server.dark @@ -2,11 +2,16 @@ /// /// Run with: `dark serve Darklang.Matter.router --port 8080` /// -/// A relay is a DUMB store-and-forward for the op log: it holds ops and hands them back -/// out; it never folds or resolves them, so it never runs anyone's code. A client that -/// pulls for review sees every op before it lands, so the worst a relay can do is -/// withhold one. It reuses the SCM.Wire codec, so the relay's own store IS an op log like -/// any instance's. +/// The server holds the op log and hands it back out: `/sync/pull` pages it, `/seed/*.db` cuts +/// it as a file. It reuses the SCM.Wire codec, so the server's own store IS an op log like any +/// instance's. +/// +/// It also FOLDS what is pushed to it, into main, like any other instance. That is what makes +/// `/m`, `/p` and the seed routes show hosted packages rather than only what this binary shipped +/// with. It is also a deferred decision, not a settled one: names bind last-writer-wins over the +/// whole store, `Darklang.Matter.router` among them, so anyone who can push here can change what +/// this server resolves. The write secret is the only thing standing in front of that. See +/// `LibDB.Inserts` for what the old `effective = 0` defended and why it went. /// /// `handlers`, near the bottom, is the route table. module Darklang.Matter @@ -303,9 +308,9 @@ let ownersHandlerFn (_req: HttpRequest) : HttpResponse = page "Publishers" - ("

Publishers

Who owns the packages this relay serves. Code pushed here by " - ++ "other machines is held and handed back on request, but it is not folded into what the relay runs, " - ++ "so it does not appear here. Why.

" + ("

Publishers

Who owns the packages this server serves. Code pushed here " + ++ "is folded into main like anywhere else, so it shows up here as soon as it arrives. " + ++ "What that means.

" ++ $"
{rows}
") @@ -334,13 +339,12 @@ let statsHandlerFn (_req: HttpRequest) : HttpResponse = ++ $"
{opsStat}{fnStat}{typeStat}{valueStat}{edgeStat}
" ++ "

Two numbers, two different things

" ++ "

The op count is everything anyone has ever pushed here. The function, type and " - ++ "value counts are smaller and mean something else: they are what this relay itself RUNS and can " - ++ "therefore show you source for.

" - ++ "

A relay stores what you push without folding it into its own package set. That " - ++ "is deliberate, and it is the whole reason a relay is safe to point a machine at: pushing code here " - ++ "cannot change what this server runs, so nobody can push a new version of the relay to the relay. " - ++ "The cost is that hosted code is held, served back to whoever pulls it, and not browsable on these " - ++ "pages. What you can browse here is the package set this build ships with.

" + ++ "value counts are smaller and mean something else: they are what is LIVE, the latest version of " + ++ "each name rather than every version of it.

" + ++ "

Pushed code is folded into main here, the same as on your own machine, which " + ++ "is why you can browse it on these pages and fetch it as a seed. It also means a push can rebind " + ++ "a name this server itself resolves, including this page. Who may bind what is not answered yet; " + ++ "for now the write secret is the whole of the answer, so treat push access as trust.

" ++ "

What an op is

" ++ "

One recorded change: publishing a function, moving a name, deprecating " ++ "something. Every version of everything is still here, which is why the op count runs so far ahead " @@ -362,7 +366,10 @@ val handlers = post "/branch/push" Darklang.Matter.Relay.branchPushHandlerFn get "/branch/pull" Darklang.Matter.Relay.branchPullHandlerFn get "/branch/list" Darklang.Matter.Relay.branchListHandlerFn - get "/branch/owners" Darklang.Matter.Relay.branchOwnersHandlerFn ] + get "/branch/owners" Darklang.Matter.Relay.branchOwnersHandlerFn + // Before the `/seed/:file` route below it, which would otherwise swallow `meta` as a filename. + get "/seed/meta" Darklang.Matter.Relay.seedMetaHandlerFn + get Darklang.Matter.Relay.seedRoute Darklang.Matter.Relay.seedHandlerFn ] /// One server, two audiences. /// diff --git a/scripts/testing/_gates-sync b/scripts/testing/_gates-sync index c0fc3988e6..29a61ec63f 100644 --- a/scripts/testing/_gates-sync +++ b/scripts/testing/_gates-sync @@ -192,8 +192,52 @@ gate_server_folds() { fail "$draftable hosted op(s) look like this store's draft; a discard would delete them" fi + # Folded is not the same as RUN. A `val` body is guest code; folding it writes a projection row + # and executes nothing, and on a server that distinction is the difference between holding + # somebody's code and running it. Nothing else in the suite can ask this: it needs a real push + # from another store, and then a RESTART, because the startup grow is what would evaluate it. + cli val 'Tests.ServerFolds.hosted' '7L' > /dev/null 2>&1 + cli commit "server-folds hosted val" -y > /dev/null 2>&1 + cli push "http://localhost:$PORT" > /dev/null 2>&1 + + val_rows=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'hosted' AND modules LIKE '%ServerFolds%';") + [[ "$val_rows" == "1" ]] || fail "the pushed val did not fold into the server's projection" + + unevaluated() { + sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_values pv + JOIN locations l ON l.item_hash = pv.hash + WHERE l.name = 'hosted' AND l.modules LIKE '%ServerFolds%' AND pv.rt_dval IS NULL;" \ + 2>/dev/null || echo -1 + } + + if [[ "$(unevaluated)" == "1" ]]; then + echo " ok a pushed val is folded but not evaluated by the push" + else + fail "the push evaluated a pushed val's body" + fi + + kill $SRV 2>/dev/null || true + wait $SRV 2>/dev/null || true + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + >> "$WORK/server/serve.log" 2>&1 & + SRV=$! + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && break + sleep 1 + done + + if [[ "$(unevaluated)" == "1" ]]; then + echo " ok and a restart does not evaluate it either" + else + fail "the startup grow evaluated a pushed val's body" + fi + [[ "$failures" == "0" ]] || exit 1 - echo "server-folds: a push reaches the server's projection, and stays out of its draft." + echo "server-folds: a push reaches the server's projection, stays out of its draft, and is never run." } gate_relay_routes() { @@ -786,3 +830,188 @@ gate_sync_multi_instance() { echo "passed: $pass failed: $fail" [[ "$fail" -eq 0 ]] } + +# --- seeds: the store as a file, cut at a commit ------------------------------------------------ +# Crosses a process AND a machine boundary: a real server on a port cuts a seed from its own store, +# a client fetches it over HTTP and becomes a store from it. Nothing in-process can answer the +# question, which is whether what CI fetches is fixed by the COMMIT it names rather than by when it +# asked. That is the property a pin rests on, so it is tested by moving the store underneath and +# asking again. +gate_seed_serving() { + set -euo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + WORK=rundir/seed-serving-test + PORT=${PORT:-9094} + SECRET='seed-serving-secret' + + rm -rf "$WORK" + mkdir -p "$WORK/server/logs" "$WORK/client/logs" "$WORK/fresh" "$WORK/fresh2" + scripts/testing/_copy-store "$WORK/server/data.db" + scripts/testing/_copy-store "$WORK/client/data.db" + + cli() { env DARK_CONFIG_RUNDIR="$PWD/$WORK/client/" HOME="$PWD/$WORK/client" "$PWD/$CLI" "$@"; } + + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" relay_grants "$PWD/$CLI" "$PORT" + # Cutting a seed WRITES a file and opens SQLite on it, so the server needs those two beyond what + # `relay_grants` gives a read-only relay. Scoped to its own rundir: the cache lives beside the store. + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" "$PWD/$CLI" permissions allow file read+write "$PWD/$WORK/server" > /dev/null 2>&1 + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" "$PWD/$CLI" permissions allow native > /dev/null 2>&1 + cli permissions allow package-write > /dev/null 2>&1 + + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + > "$WORK/server/serve.log" 2>&1 & + SRV=$! + trap 'kill $SRV 2>/dev/null' EXIT + + up=false + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && { up=true; break; } + sleep 1 + done + [[ "$up" == "true" ]] || { + echo "the server never came up on $PORT (see $WORK/server/serve.log)" >&2 + exit 2 + } + + U="http://localhost:$PORT" + failures=0 + fail() { echo " FAIL $1" >&2; failures=$((failures + 1)); } + + # `/seed/meta` exists so a pin can be CHECKED without downloading the seed. If it needed the + # seed to answer, every "has anything moved?" would cost twelve megabytes. + meta=$(curl -s --max-time 15 "$U/seed/meta") + field() { printf '%s' "$meta" | sed -n "s/.*\"$1\":\"\\?\\([^,\"}]*\\).*/\\1/p"; } + HEAD1=$(field head) + [[ -n "$HEAD1" ]] || fail "/seed/meta named no head commit: $meta" + [[ "$(field format)" == "1" ]] || fail "/seed/meta reported format $(field format), not 1" + [[ "$(field ops)" -gt 0 ]] || fail "/seed/meta reported no ops: $meta" + echo " ok /seed/meta answers without cutting a seed ($(field ops) ops at ${HEAD1:0:8})" + + fetch() { # fetch + local dir="$1" commit="$2" args=() + [[ "$commit" == "latest" ]] || args=(--commit "$commit") + env DARK_CONFIG_RUNDIR="$PWD/$dir/" scripts/fetch-seed --url "$U" "${args[@]}" --force \ + > "$WORK/fetch.log" 2>&1 \ + || { echo " FAIL the fetch at $commit failed; the server said:" >&2 + curl -s --max-time 60 "$U/seed/$commit.db" | head -c 400 >&2; echo >&2; return 1; } + } + ops_in() { sqlite3 "$1" "SELECT count(*) FROM package_ops;" 2>/dev/null || echo -1; } + + fetch "$WORK/fresh" "$HEAD1" + SEED1="$WORK/fresh/seed.db" + [[ -f "$SEED1" ]] || { echo "the fetch produced no seed (see above)" >&2; exit 2; } + + # A seed is ONE file. Served in WAL mode, the `.db` alone is not the whole database, so whether a + # fetch got everything would depend on when a checkpoint happened to run. + if [[ -f "$SEED1-wal" ]]; then + fail "the served seed came with a -wal, so the file is not the whole database" + else + echo " ok the served seed is a single settled file" + fi + + cut_at=$(sqlite3 "$SEED1" "SELECT value FROM store_meta WHERE key = 'cut_at';" 2>/dev/null || echo "") + if [[ "$cut_at" == "$HEAD1" ]]; then + echo " ok and it says which commit it is a cut of" + else + fail "the seed says it was cut at '$cut_at', but it was asked for $HEAD1" + fi + + N1=$(ops_in "$SEED1") + + # The seed has to be a usable STORE, not just a well-formed file: a fresh install folds it and + # then resolves names out of it. This is the whole point of serving one. + env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ + "$PWD/$CLI" permissions allow package-read > /dev/null 2>&1 || true + got=$(env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ + "$PWD/$CLI" eval '(Stdlib.List.length [1L, 2L, 3L]) |> Stdlib.Int64.toString' 2>&1 | tail -1 || true) + if [[ "$got" == *3* ]]; then + echo " ok a store grown from the fetched seed resolves names and evaluates" + else + fail "a store grown from the fetched seed could not evaluate: $got" + fi + + # Now move the server's store underneath the pin, by pushing new work to it. + cli fn 'Tests.SeedServing.after' '() : String = "afterthepin"' > /dev/null 2>&1 + cli commit "seed-serving gate" -y > /dev/null 2>&1 + cli connect "$U" --secret "$SECRET" > /dev/null 2>&1 + cli push "$U" > /dev/null 2>&1 + + HEAD2=$(curl -s --max-time 15 "$U/seed/meta" | sed -n 's/.*"head":"\([^"]*\)".*/\1/p' || true) + if [[ -n "$HEAD2" && "$HEAD2" != "$HEAD1" ]]; then + echo " ok the push moved the server's head (${HEAD1:0:8} -> ${HEAD2:0:8})" + else + fail "the push did not move the server's head, so the rest of this gate proves nothing" + exit 1 + fi + + # The cache is deleted first, so the cut is RECOMPUTED against a store that has moved. Served + # from cache the next assertion would pass for the wrong reason. + rm -rf "$WORK/server/seeds" + + fetch "$WORK/fresh2" "$HEAD1" + SEED2="$WORK/fresh2/seed.db" + N2=$(ops_in "$SEED2") + if [[ "$N2" == "$N1" ]]; then + echo " ok re-cut at the same commit after the push: same $N1 ops" + else + fail "the same commit cut $N1 ops before the push and $N2 after; a pin is not reproducible" + fi + + has_commit=$(sqlite3 "$SEED2" "SELECT count(*) FROM commits WHERE hash = '$HEAD2';" 2>/dev/null || echo -1) + if [[ "$has_commit" == "0" ]]; then + echo " ok and it does not carry work committed after it" + else + fail "a seed cut at $HEAD1 carries the later commit $HEAD2" + fi + + fetch "$WORK/fresh2" latest + N3=$(ops_in "$SEED2") + if [[ "$N3" -gt "$N1" ]]; then + echo " ok /seed/latest.db moves with the store ($N1 -> $N3 ops)" + else + fail "/seed/latest.db still cuts $N3 ops after a push that added some" + fi + + # The PIN, end to end, against a scratch copy of `package-set.txt` so the tree's own is untouched. + # This is the closest thing to a test of 8.C there can be from here: the real path runs against a + # deployed server, and nothing deploys from this clone. + cp package-set.txt "$WORK/package-set.txt" + export DARK_PACKAGE_SET="$PWD/$WORK/package-set.txt" + + if scripts/packages/pin head --url "$U" > "$WORK/pin.log" 2>&1; then + pinned=$(awk '$1 == "commit" { print $2 }' "$DARK_PACKAGE_SET") + if [[ "$pinned" == "$HEAD2" ]]; then + echo " ok \`pin head\` records the server's head, checked before writing it" + else + fail "pin wrote '$pinned', but the server's head is $HEAD2" + fi + else + fail "pin head failed: $(cat "$WORK/pin.log")" + fi + + # A commit the server does not have is refused AT PIN TIME. Without this the pin is only + # discovered to be wrong at the next CI run, by which time nobody is looking. + if scripts/packages/pin 0000000000000000 --url "$U" > "$WORK/pin.log" 2>&1; then + fail "pin accepted a commit the server does not have" + else + echo " ok and refuses a commit the server does not have" + fi + + # With a pin in place, a fetch that names no commit takes the PIN rather than the head. That is + # the whole behaviour CI depends on. + scripts/packages/pin "$HEAD1" --url "$U" > "$WORK/pin.log" 2>&1 || fail "could not pin back to $HEAD1" + rm -f "$WORK/fresh2/seed.db" + fetch "$WORK/fresh2" latest + if [[ "$(ops_in "$SEED2")" == "$N1" ]]; then + echo " ok and a fetch with no commit follows the pin, not the head" + else + fail "a pinned fetch got $(ops_in "$SEED2") ops; the pin at ${HEAD1:0:8} has $N1" + fi + unset DARK_PACKAGE_SET + + [[ "$failures" == "0" ]] || exit 1 + echo "seed-serving: a seed is fixed by the commit it names, follows the pin, and grows into a working store." +} diff --git a/scripts/testing/gates b/scripts/testing/gates index e10f79ef92..7fb8096663 100755 --- a/scripts/testing/gates +++ b/scripts/testing/gates @@ -24,6 +24,7 @@ GATES=( setup relay-routes server-folds + seed-serving sync-hostile-relay sync-multi-instance lsp-branches @@ -38,14 +39,15 @@ GATES=( # What CI runs, in one config line: the three gates it has always run, the ones that need only # the debug build already in backend/Build. first-day wants a published artifact and the sync # gates add minutes; those run locally, by name or via `all`. -CI_GATES=(relay-routes server-folds workbench-scm workbench-views) +CI_GATES=(relay-routes server-folds seed-serving workbench-scm workbench-views) describe() { # shellcheck disable=SC2016 # the backticks are prose, not expansion case "$1" in setup) echo '`dark sync setup`, end to end, own rundir' ;; relay-routes) echo "the relay's HTTP surface, given bad input" ;; - server-folds) echo "a push reaches the server's projection, and stays out of its draft" ;; + server-folds) echo "a push folds into the server's projection, and is never run there" ;; + seed-serving) echo 'a seed cut at a commit, fetched over HTTP, grown into a store' ;; sync-hostile-relay) echo 'the sync/branch CLIENTS, given a relay that lies' ;; sync-multi-instance) echo 'four instances, branches, review queues, agents' ;; lsp-branches) echo 'the `dark/*` branch surface, as an editor drives it' ;; From 1b952580f883c84a309ffc461598d4d49bb77a26 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 11:57:57 -0400 Subject: [PATCH 12/32] Pin the package set to a commit, and land it switched off `package-set.txt` records which package set this kernel was tested against. Two fields, `commit` and `format`, and deliberately no URL: a pin says WHICH package set, not where to get it, two mirrors serving the same commit are interchangeable, and an address in the repo is a deployment detail that ages badly. It comes from `DARK_SEED_URL` or `--url`. It ships `commit unset`, which every reader treats as "build the package set from packages/" -- exactly what happens today. So this is a mechanism with a one-line switch rather than a cutover, which is the only responsible way to land a half that has never run against a deployed server. `scripts/packages/pin head | | --show | --unset` writes it, and checks every pin before writing: `/seed/meta?commit=` makes the server resolve the commit and answer what a cut at it would carry, so a bad pin is refused at pin time instead of at the next CI run. `scripts/build/prepare-package-set` is the one place that answers "where does this build's package set come from", and CI's three package-reloading jobs call it instead of `reload-packages`. It does NOT fall back to reloading when the server is unreachable: a build that silently used the tree instead of the pin is the failure the pin exists to prevent. `seed-db` gained a cache keyed on the pin rather than on the week, because a pinned package set is immutable. `scripts/run-local-exec refs generate` is the piece that makes a fetched seed usable: the kernel resolves its entry points by hash, and that file is a projection of the store, not something the seed carries. The fill path wrote it as a side effect of reloading; this is the same step for a store that has no packages/ to reload. Against the plan, which wanted `package-ref-hashes.txt` tracked in git. It should not be, and this is what makes that the better answer: it is derived, tracking it taxes every rebase (two branches that both touch packages conflict, and GitHub cannot regenerate it), and the endgame reason for tracking -- no packages/ to regenerate from -- is answered by regenerating from the pinned store instead. What git records is one line of pin. --- .circleci/config.yml | 33 +++++++++--- AGENTS.md | 5 ++ package-set.txt | 20 +++++++ scripts/build/prepare-package-set | 42 +++++++++++++++ scripts/fetch-seed | 38 ++++++++++++- scripts/packages/pin | 90 +++++++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 package-set.txt create mode 100755 scripts/build/prepare-package-set create mode 100755 scripts/packages/pin diff --git a/.circleci/config.yml b/.circleci/config.yml index 1daa55588b..d6cde90d95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -228,8 +228,8 @@ jobs: name: Run migrations to create database command: scripts/run-local-exec migrations run - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - run: name: Export the seed the publish embeds command: scripts/run-local-exec export-seed rundir/seed.db @@ -315,8 +315,8 @@ jobs: name: Run migrations to create database command: scripts/run-local-exec migrations run - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - assert-clean-worktree # Just the host build, on main as much as anywhere else. This is a @@ -379,14 +379,31 @@ jobs: - run: name: Run migrations to create database command: scripts/run-local-exec migrations run + # Keyed on the PIN, not on the week: a pinned package set is immutable, so this hits on every + # release build until someone re-pins. Costs nothing when there is no pin -- `package-set.txt` + # then says `commit unset`, the key is stable, and the cached rundir is simply not used + # because the step below reloads instead. + - restore_cache: + keys: + - v1-seed-{{ checksum "package-set.txt" }} - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - run: name: Export the seed command: | - sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true - scripts/run-local-exec export-seed rundir/seed.db + # A no-op on the pinned path, where the fetched file already IS the seed and is settled + # out of WAL. On the reload path the store is live and this is the cut. + if [[ -f rundir/seed.db ]] && [[ "$(awk '$1 == "commit" { print $2 }' package-set.txt)" != "unset" ]]; then + echo "using the fetched seed at the pin" + else + sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true + scripts/run-local-exec export-seed rundir/seed.db + fi + - save_cache: + paths: + - rundir/seed.db + key: v1-seed-{{ checksum "package-set.txt" }} - persist_to_workspace: root: "." paths: diff --git a/AGENTS.md b/AGENTS.md index 9b84febb4b..0f20f8aa85 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -326,6 +326,11 @@ Empty is tolerated; non-empty with a missing key crashes at startup with "Packag hash not found". After adding a ref: `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages` +It is a PROJECTION of the store, not a source file, which is why it is not tracked. A store +that came from a seed has no `packages/` to reload, so regenerate it from the store instead: +`scripts/run-local-exec refs generate`. `scripts/build/prepare-package-set` does that on the +pinned path; see `package-set.txt`. + **Name resolution in test files.** `backend/testfiles/` is parsed with owner "Tests", so `Darklang.*` names need full qualification or the `Stdlib.` shortcut. `Stdlib.Json.ParseError.toString` and `Darklang.SCM.Branch.mainBranchId` resolve; `SCM.Branch.mainBranchId` doesn't. Impl: diff --git a/package-set.txt b/package-set.txt new file mode 100644 index 0000000000..af8a09c3bb --- /dev/null +++ b/package-set.txt @@ -0,0 +1,20 @@ +# The package set this kernel is pinned to. +# +# `scripts/packages/pin` writes this file; `scripts/fetch-seed`, CI and the dev container read it. +# Its whole job is to make "which packages" a decision recorded in git rather than whatever the +# tree happened to contain when a build ran. +# +# There is deliberately no URL here. A pin says WHICH package set, not where to get it: two +# mirrors serving the same commit are interchangeable, and a URL in the repo is a deployment +# detail that ages badly and belongs to whoever runs the server. The address comes from +# `DARK_SEED_URL` in the environment, or `--url` on the command line. +# +# commit the SCM commit on the package server that this kernel was tested against. `unset` +# means there is no pin yet, and everything that reads this file falls back to +# building the package set from `packages/` -- which is what happens today. +# format the op-blob layout the seed at that commit is written in. A fetch that gets a +# different one refuses rather than folding it, because a layout this binary has +# never seen cannot be read by trying harder. + +commit unset +format 1 diff --git a/scripts/build/prepare-package-set b/scripts/build/prepare-package-set new file mode 100755 index 0000000000..15ce6beac4 --- /dev/null +++ b/scripts/build/prepare-package-set @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Get this build's package set into the store, from whichever source `package-set.txt` names. +# +# commit unset reload `packages/` from disk, which is what has always happened +# commit fetch the seed at that commit from the package server, and generate the +# kernel's ref hashes from it +# +# One script rather than a branch in three CI jobs, because the two halves have to agree about what +# "the package set" is or a build embeds one and tests another. +# +# The pinned path needs `DARK_SEED_URL` (or `--url`). It deliberately does NOT fall back to +# reloading if the server is unreachable: a build that silently used the tree instead of the pin is +# the exact failure the pin exists to prevent. + +set -euo pipefail + +[[ -f package-set.txt ]] || { echo "prepare-package-set: run from the repo root" >&2; exit 2; } + +URL_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --url) URL_ARGS=(--url "${2:?--url needs a url}"); shift 2 ;; + -h|--help) sed -n '3,15p' "$0" | sed 's/^# \?//'; exit 0 ;; + *) echo "prepare-package-set: unknown argument $1" >&2; exit 2 ;; + esac +done + +PIN=$(awk '$1 == "commit" { print $2 }' package-set.txt) + +if [[ -z "$PIN" || "$PIN" == "unset" ]]; then + echo "No pin in package-set.txt; building the package set from packages/." + exec scripts/build/reload-packages +fi + +echo "Pinned to ${PIN}; fetching the seed rather than reloading packages/." +scripts/fetch-seed "${URL_ARGS[@]}" --force + +# The kernel resolves its entry points by HASH, and that file is not in the seed -- it is a +# projection of it. Without this step every kernel lookup resolves to "" and the binary fails on +# its first command, somewhere that says nothing about why. +scripts/run-local-exec refs generate diff --git a/scripts/fetch-seed b/scripts/fetch-seed index 0dd6f62be1..834b9df8c3 100755 --- a/scripts/fetch-seed +++ b/scripts/fetch-seed @@ -3,10 +3,14 @@ # Get a package seed, from a file or from a server, and install it as this rundir's store. # # scripts/fetch-seed --from a seed already on disk -# scripts/fetch-seed --url that server's current main +# scripts/fetch-seed --url that server's main at the PIN, or its head # scripts/fetch-seed --url --commit that server's main as of # scripts/fetch-seed DARK_SEED_URL, or the built-in default # +# With no --commit, the commit comes from `package-set.txt`; `commit unset` there means no pin, +# and the fetch takes the server's head. A pinned fetch also CHECKS the seed's format against the +# pin, because a layout this build has never seen cannot be read by trying harder. +# # A seed is ops, not projections, so the first open folds it (~seconds) and every store built # from the same seed agrees on every id. # @@ -29,6 +33,16 @@ URL="" COMMIT="" FORCE=false +# The pin, if this tree has one. Read before the arguments so an explicit --commit overrides it. +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" +PINNED_COMMIT="" +PINNED_FORMAT="" +if [[ -f "$PINFILE" ]]; then + PINNED_COMMIT=$(awk '$1 == "commit" { print $2 }' "$PINFILE") + PINNED_FORMAT=$(awk '$1 == "format" { print $2 }' "$PINFILE") + if [[ "$PINNED_COMMIT" == "unset" ]]; then PINNED_COMMIT=""; fi +fi + while [[ $# -gt 0 ]]; do case "$1" in --from) FROM="${2:?--from needs a path}"; shift 2 ;; @@ -58,6 +72,13 @@ else # `/seed/.db` is the route a server serves a pinned cut from; without a commit, its # current main. A bare url with no path is treated as already naming the seed, which is what # the R2 fallback is. + # An explicit --commit beats the pin; the pin beats the server's head. So a build in a pinned + # tree fetches the pinned set without anyone having to remember to ask for it. + if [[ -z "$COMMIT" && -n "$PINNED_COMMIT" ]]; then + COMMIT="$PINNED_COMMIT" + echo "Using the pin from ${PINFILE}: ${COMMIT}" + fi + if [[ -n "$COMMIT" ]]; then SEED_URL="${URL:-$DEFAULT_URL}/seed/${COMMIT}.db" elif [[ -n "$URL" ]]; then @@ -73,6 +94,21 @@ fi SEED_SIZE=$(du -h "$DEST_SEED" | cut -f1) echo "Seed at ${DEST_SEED} (${SEED_SIZE})" +# Refuse a seed in a layout the pin did not expect, BEFORE installing it. A store is the only copy +# of the ops in it, so the failure has to happen while the old one is still there. +if [[ -n "$PINNED_FORMAT" ]] && command -v sqlite3 > /dev/null 2>&1; then + GOT_FORMAT=$(sqlite3 "$DEST_SEED" "SELECT value FROM store_meta WHERE key = 'format';" 2>/dev/null || echo "") + if [[ -n "$GOT_FORMAT" && "$GOT_FORMAT" != "$PINNED_FORMAT" ]]; then + { + echo + echo "That seed is in format ${GOT_FORMAT}; ${PINFILE} pins format ${PINNED_FORMAT}." + echo "Not installing it. Either the server moved formats or this tree's pin is stale;" + echo "re-pin with scripts/packages/pin, on a build that reads the new format." + } >&2 + exit 1 + fi +fi + if [[ -f "$DEST_DB" && "$FORCE" != "true" ]]; then { echo diff --git a/scripts/packages/pin b/scripts/packages/pin new file mode 100755 index 0000000000..aa5e6b9396 --- /dev/null +++ b/scripts/packages/pin @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Pin the package set to a commit on a package server, and record it in `package-set.txt`. +# +# scripts/packages/pin head pin to what that server has right now +# scripts/packages/pin pin to a named commit (a short prefix is fine) +# scripts/packages/pin --show what is pinned now +# scripts/packages/pin --unset drop the pin, back to building from `packages/` +# +# The server comes from `--url` or `DARK_SEED_URL`; see `package-set.txt` for why it is not in +# the repo. +# +# Every pin is CHECKED before it is written, through `/seed/meta?commit=`: the server resolves the +# commit and answers what a cut at it would carry. Two counts rather than twelve megabytes, so +# checking a pin is cheap enough to do every time. A commit that does not resolve is refused here +# rather than at the next CI run. + +set -euo pipefail + +# `DARK_PACKAGE_SET` points this at another copy, which is how the gates exercise a pin without +# writing to the tree's own. +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" + +[[ -f "$PINFILE" ]] || { echo "pin: no $PINFILE (run from the repo root?)" >&2; exit 2; } +URL="${DARK_SEED_URL:-}" +WANT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --url) URL="${2:?--url needs a url}"; shift 2 ;; + --show) sed -n 's/^\(commit\|format\) */\1 /p' "$PINFILE"; exit 0 ;; + --unset) WANT="unset"; shift ;; + -h|--help) sed -n '3,17p' "$0" | sed 's/^# \?//'; exit 0 ;; + -*) echo "pin: unknown argument $1" >&2; exit 2 ;; + *) WANT="$1"; shift ;; + esac +done + +# Rewrite one `key value` line in place, leaving the comments that explain it alone. +set_field() { + local key="$1" value="$2" + local tmp + tmp=$(mktemp) + awk -v k="$key" -v v="$value" ' + $1 == k { printf "%s %s\n", k, v; next } + { print } + ' "$PINFILE" > "$tmp" + mv "$tmp" "$PINFILE" +} + +if [[ "$WANT" == "unset" ]]; then + set_field commit unset + echo "Pin dropped. The package set is built from packages/ again." + exit 0 +fi + +[[ -n "$WANT" ]] || { echo "pin: say which commit, or \`head\`, or \`--show\`" >&2; exit 2; } + +if [[ -z "$URL" ]]; then + echo "pin: no package server. Pass --url, or set DARK_SEED_URL." >&2 + exit 2 +fi + +query="$URL/seed/meta" +[[ "$WANT" == "head" ]] || query="$URL/seed/meta?commit=$WANT" + +body=$(mktemp) +code=$(curl -s -o "$body" -w '%{http_code}' --max-time 30 "$query" || echo 000) + +if [[ "$code" != "200" ]]; then + echo "pin: $URL would not pin $WANT (HTTP $code):" >&2 + sed 's/^/ /' "$body" >&2 + exit 1 +fi + +field() { sed -n "s/.*\"$1\":\"\\?\\([^,\"}]*\\).*/\\1/p" "$body"; } +COMMIT=$(field head) +FORMAT=$(field format) +OPS=$(field ops) + +[[ -n "$COMMIT" ]] || { echo "pin: $URL named no commit: $(cat "$body")" >&2; exit 1; } + +# The format is pinned alongside the commit, not just reported: a seed in a layout this kernel +# cannot read is a fetch that must fail before it folds anything, and the only way to know that +# before fetching is to have written down what was expected. +set_field commit "$COMMIT" +set_field format "$FORMAT" + +echo "Pinned to $COMMIT (format $FORMAT, $OPS ops)." +echo "Commit package-set.txt: the pin is the record of which package set this kernel was tested against." From 20fc9c34a4fcfee84f7fa3c8deedd2d8d154a46c Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 12:32:38 -0400 Subject: [PATCH 13/32] Carry a store forward when the op-log format changes An op's id comes from `Hashing.computeOpRowId` over the decoded op, not over its bytes, and that divides format changes cleanly: - a change under LibSerialization/Binary/* moves the LAYOUT and nothing's identity. The migration is a blob rewrite: decode with the old reader, re-encode with the new writer, every id untouched, projections re-folded. That is `dark store upgrade`. - a change under LibSerialization/Hashing/* moves every op id, item hash and commit hash at once, which is a whole-log re-mint. Not built. StoreUpgrade.fs ends with the six steps it needs and the two properties that make it worth the care. The migrator refuses rather than guessing which it faces: it recomputes each op's id from the decoded op and compares it with the id the store filed it under. Disagreement means the store's ids were minted by different hashing, and a blob rewrite would be a lie. The gate forges exactly that, which is the most important assertion in it. The backup lands first, through the online-backup API, so the rollback target exists before anything is touched. The rewrite and the format stamp are one transaction, through `executeTransactionSync` rather than hand-written BEGIN/COMMIT: connections are pooled, so a BEGIN and the statements after it are not guaranteed to be on the same one, and the transaction would silently cover nothing. `validateVersion` accepts any version up to this build's and refuses only newer. An old blob can be read because every historical reader stays in the binary; a newer one cannot be read by trying harder. A build reading an OLDER format than the store gets a note, not a refusal, and the note names a file to move rather than a verb to run. Refusing to open would refuse the rollback too, and the projections hold newer blobs as well, so such a build dies resolving the name of whatever command you typed. A `mv` needs no working binary. The note comes from `growIfNeeded`, which every process that opens the store passes through; the first version of it lived only in `Releases.runPending`, which a dev build never reaches, and the gate is what showed that. `DARK_FORMAT_VERSION` is the synthetic bump the gate runs on. One build writes the version named and reads every version up to it, so the layout is identical and a migration across it is exactly the format-only case. It is the only way to exercise a migrator while one format exists, and having it mechanised before the first real bump is the point. Also a serialization corpus, `serialization-artifacts/corpus/v/`, ten types, never regenerated. Different question from the golden files beside it: a golden is regenerated whenever a format change is intended, so the moment the format moves the old bytes are gone and nothing checks they can still be read. This decodes a blob a past build wrote, re-encodes with today's writer, decodes that, and compares, which is the migration's core operation. At the current version it also asserts byte-exactness, as a prefix so a test value can be appended without destroying history; that catches a layout change made without bumping the version, which is the one format bug that corrupts silently. `store-upgrade` goes in the CI gate subset rather than staying local-only. It is the migrator's only test, and a migration that has quietly stopped working is discovered at the worst possible moment. --- .../Builtins/Builtins.Matter/Libs/PM/Store.fs | 77 ++++++ backend/src/LibDB/LibDB.fsproj | 1 + backend/src/LibDB/Releases.fs | 42 +++- backend/src/LibDB/Seed.fs | 14 +- backend/src/LibDB/StoreUpgrade.fs | 227 ++++++++++++++++++ .../src/LibSerialization/Binary/BaseFormat.fs | 42 +++- .../LibSerialization/Binary/Serialization.fs | 4 +- .../corpus/v1/packageLocation.bin | Bin 0 -> 106 bytes .../corpus/v1/packageOp.bin | Bin 0 -> 5175 bytes .../corpus/v1/ptPackageFn.bin | Bin 0 -> 3100 bytes .../corpus/v1/ptPackageType.bin | Bin 0 -> 198 bytes .../corpus/v1/ptPackageValue.bin | Bin 0 -> 281 bytes .../corpus/v1/rtDval.bin | Bin 0 -> 39487 bytes .../corpus/v1/rtInstructions.bin | Bin 0 -> 73 bytes .../corpus/v1/rtPackageFn.bin | Bin 0 -> 104 bytes .../corpus/v1/rtPackageValue.bin | Bin 0 -> 89 bytes .../corpus/v1/toplevel.bin | Bin 0 -> 153 bytes .../tests/Tests/Serialization.Binary.Tests.fs | 215 ++++++++++++++++- packages/darklang/cli/registry.dark | 3 +- packages/darklang/cli/store.dark | 145 +++++++++++ packages/darklang/stdlib/localStore.dark | 49 ++++ scripts/testing/_gates-store | 97 ++++++++ scripts/testing/_gates-sync | 2 +- scripts/testing/gates | 13 +- 24 files changed, 902 insertions(+), 29 deletions(-) create mode 100644 backend/src/LibDB/StoreUpgrade.fs create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/rtPackageFn.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/rtPackageValue.bin create mode 100644 backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin create mode 100644 packages/darklang/cli/store.dark diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs index 19d8d0a1a1..564209247b 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs @@ -12,6 +12,7 @@ open LibExecution.Effects open LibExecution.Builtin.Shortcuts module Dval = LibExecution.Dval +module VT = LibExecution.ValueType /// Refuse a store-level operation unless every non-root frame is bundled Darklang @@ -101,6 +102,82 @@ let fns () : List = callEffects = set [ Effect.PackageWrite ] deprecated = NotDeprecated } + // Move the store to the op-log format this build writes, and put back the copy that made. + // + // First-party only, for the same reason as backup/restore: this rewrites every op blob in the + // log and drops every projection. A guest holding package-write must not reach it through a + // wrapper. + // + // A no-op in practice until the first real format bump -- `from` and `to` are equal, and it + // says so rather than doing anything. The mechanism exists now so the bump is not also the + // first time the migration runs. + { name = fn "pmStoreUpgrade" 0 + typeParams = [] + parameters = [ Param.make "unit" TUnit "" ] + returnType = + TypeReference.result + (TTuple(TInt64, TInt64, [ TInt64; TInt64; TString ])) + TString + description = + "Rewrites this store's op log into the format this build writes. Ok is (from, to, rewritten, unreadable, backupPath)." + fn = + let okKT = + KTTuple( + VT.known KTInt64, + VT.known KTInt64, + [ VT.known KTInt64; VT.known KTInt64; VT.known KTString ] + ) + (function + | state, vm, _, [| DUnit |] -> + uply { + requireBundledCaller state vm "pmStoreUpgrade" + let! result = LibDB.StoreUpgrade.upgrade () + match result with + | Ok r -> + return + Dval.resultOk + okKT + KTString + (DTuple( + DInt64(int64 r.from), + DInt64(int64 r.to_), + [ DInt64(int64 r.rewritten) + DInt64(int64 r.unreadable) + DString r.backup ] + )) + | Error e -> return Dval.resultError okKT KTString (DString e) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + // `Native` alongside the writes: this opens SQLite directly to run the rewrite in one + // transaction, which a path rule alone cannot confine. + callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ] + deprecated = NotDeprecated } + + { name = fn "pmStoreRollback" 0 + typeParams = [] + parameters = + [ Param.make "target" TInt64 "the format version that was upgraded TO" ] + returnType = TypeReference.result TString TString + description = + "Restores the copy `pmStoreUpgrade` took on its way to . Ok is the path restored from." + fn = + (function + | state, vm, _, [| DInt64 target |] -> + uply { + requireBundledCaller state vm "pmStoreRollback" + let! result = LibDB.StoreUpgrade.rollback (uint32 target) + match result with + | Ok path -> return Dval.resultOk KTString KTString (DString path) + | Error e -> return Dval.resultError KTString KTString (DString e) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ] + deprecated = NotDeprecated } + // Whether a write secret is stored for a relay, WITHOUT handing it over. // // `dark sync setup` needs to know if one is already there, so that pressing Enter keeps it rather diff --git a/backend/src/LibDB/LibDB.fsproj b/backend/src/LibDB/LibDB.fsproj index b8d213c233..3ee532e216 100644 --- a/backend/src/LibDB/LibDB.fsproj +++ b/backend/src/LibDB/LibDB.fsproj @@ -51,6 +51,7 @@ + diff --git a/backend/src/LibDB/Releases.fs b/backend/src/LibDB/Releases.fs index 8016707b8d..ca35f18d7a 100644 --- a/backend/src/LibDB/Releases.fs +++ b/backend/src/LibDB/Releases.fs @@ -267,19 +267,41 @@ let storedFormat () : Option = | false, _ -> None) -/// Refuse a store written by a NEWER build's format, and stamp one that carries no format yet. +/// Say so when a store was written by a NEWER build's format, and stamp one that carries no format +/// yet. /// /// The asymmetry is the point. A store BEHIND this build is the migrator's job (it can read an old -/// layout, because every historical reader stays in the binary). A store AHEAD of it is not -/// recoverable by reading harder: the layout is one this binary has never seen, and guessing at it -/// corrupts the only copy of the ops. +/// layout, because every historical reader stays in the binary). A store AHEAD of it cannot be read +/// by trying harder: the layout is one this binary has never seen. +/// +/// SAID, not raised, and the wording matters more than usual, because there is no working command +/// left to recover WITH. The projections hold blobs in the newer layout too, so this build dies on +/// its first package lookup -- which includes resolving the name of the command you typed. So the +/// note names the FILE to move, not a verb to run: a `mv` needs no working binary. +/// +/// `dark store rollback` covers the other case, and the likelier one: you upgraded, you are still +/// on the build that did it, and you want it undone. +let noteFormatSkew () : unit = + match storedFormat () with + | Some n when n > LibSerialization.Binary.BaseFormat.currentVersion -> + System.Console.Error.WriteLine( + $"note: this store is format {n} and this build reads " + + $"{LibSerialization.Binary.BaseFormat.currentVersion}, so its ops cannot be read. Upgrade " + + $"the binary. If this store was upgraded here, the copy from before that is at " + + $"{Sqlite.currentDbPath}.pre-v{n} -- move it back over {Sqlite.currentDbPath}." + ) + | _ -> () + + +/// `noteFormatSkew`, plus the stamp for a store that carries none. +/// +/// Stamping is a WRITE, so it belongs here in the migration path rather than on every open: a store +/// that has run this once carries the stamp from then on. let private checkFormat () : unit = + noteFormatSkew () + match storedFormat () with - | Some n when n > LibSerialization.Binary.BaseFormat.CurrentVersion -> - Exception.raiseInternal - "this store was written by a newer Darklang than this one and cannot be read safely" - [ "store format", string n - "this build reads", string LibSerialization.Binary.BaseFormat.CurrentVersion ] + | Some n when n > LibSerialization.Binary.BaseFormat.currentVersion -> () | _ -> // Stamp it, so from here every store says what it is. `INSERT OR REPLACE` rather than a // conditional: the value is the same whether the row was missing or already right. @@ -289,7 +311,7 @@ let private checkFormat () : unit = Sql.query "INSERT OR REPLACE INTO store_meta (key, value) VALUES ('format', @v)" |> Sql.parameters - [ "v", Sql.string (string LibSerialization.Binary.BaseFormat.CurrentVersion) ] + [ "v", Sql.string (string LibSerialization.Binary.BaseFormat.currentVersion) ] |> Sql.executeStatementSync diff --git a/backend/src/LibDB/Seed.fs b/backend/src/LibDB/Seed.fs index aecbe5db87..bcdc40eaa9 100644 --- a/backend/src/LibDB/Seed.fs +++ b/backend/src/LibDB/Seed.fs @@ -35,6 +35,10 @@ module Permission = LibExecution.Permissions /// context for whatever the command was asked to do, not an event worth repeating before every answer. let mutable private warnedAboutUnreadableOps = false +/// Whether this process has already said that the store's format is ahead of this build's. Once, +/// for the same reason: it is context for the command, not an event. +let mutable private warnedAboutFormatSkew = false + // --------------------- // Export @@ -210,7 +214,7 @@ let exportAt (outputPath : string) (upToCommit : string option) : Task = """ stampCmd.Parameters.AddWithValue( "$format", - string LibSerialization.Binary.BaseFormat.CurrentVersion + string LibSerialization.Binary.BaseFormat.currentVersion ) |> ignore stampCmd.Parameters.AddWithValue("$cutAt", cutAt) |> ignore @@ -792,6 +796,14 @@ let growIfNeeded : Task = task { use _span = Telemetry.span "seed.growIfNeeded" [] + + // Every process that opens the store passes through here, which is the only place a skew + // between the store's format and this build's is certain to be noticed. `Releases.runPending` + // says it too, but a shipped binary reaches that only when it has an embedded seed to unpack. + if not warnedAboutFormatSkew then + warnedAboutFormatSkew <- true + Releases.noteFormatSkew () + let! appliedCount = Telemetry.timeTask "seed.applyOps" [] (fun () -> applyUnappliedOps ()) // The fold above reads effective=1 only, so branch-scoped Decisions (a branch's propagation diff --git a/backend/src/LibDB/StoreUpgrade.fs b/backend/src/LibDB/StoreUpgrade.fs new file mode 100644 index 0000000000..f0f8adad10 --- /dev/null +++ b/backend/src/LibDB/StoreUpgrade.fs @@ -0,0 +1,227 @@ +/// Moving a STORE from one op-log format to the next. +/// +/// After the flip there is no `packages/` to rebuild a store from, so a format change has to carry +/// the store forward in place. That divides into two cases with very different costs, and telling +/// them apart is the first thing this module does: +/// +/// FORMAT-ONLY the BINARY layout changed (`LibSerialization/Binary/*`). Op ids are derived from +/// the DECODED op by `Hashing.computeOpRowId`, not from its bytes, so nothing's +/// identity moves. The migration is a blob rewrite: decode with the old reader, +/// re-encode with the new writer, leave every id alone, re-fold the projections. +/// That is what this module does. +/// +/// IDENTITY the HASHING changed (`LibSerialization/Hashing/*`). Every op id, item hash and +/// commit hash moves, every reference inside an op has to be remapped, and the +/// commit chain has to be rebuilt parent-first. Not built; see the note at the +/// bottom of this file for what it needs. +/// +/// The distinction is not a judgement call, and this refuses rather than guessing: it recomputes +/// each op's id from the decoded op and compares it with the id the store has. If they disagree, +/// the store's ids were minted by a different hashing and the blob rewrite would be a lie. +module LibDB.StoreUpgrade + +open System.Threading.Tasks +open FSharp.Control.Tasks + +open Prelude + +open Fumble +open LibDB.Sqlite + +module PT = LibExecution.ProgramTypes +module BS = LibSerialization.Binary.Serialization +module BaseFormat = LibSerialization.Binary.BaseFormat +module Hashing = LibSerialization.Hashing.Hashing + + +type Report = + { + from : uint32 + to_ : uint32 + /// Ops decoded and written back out. + rewritten : int + /// Ops this build cannot read at all: a peer's newer format, stored inert. Left EXACTLY as + /// they are. Re-encoding is impossible and dropping them would lose work a later build can + /// still apply, which is the promise the log makes. + unreadable : int + /// Where the pre-migration store was copied to. + backup : string + } + + +let private storedFormat () : uint32 = + Releases.storedFormat () |> Option.defaultValue 1u + + +/// Where the pre-migration copy goes: beside the store, named for the version being moved TO. +/// +/// Named for the target and not the source so it reads as "the store from before v2", which is +/// what someone rolling back is looking for. +let backupPathFor (target : uint32) : string = + $"{Sqlite.currentDbPath}.pre-v{target}" + + +/// Move this store to the format this build writes. +/// +/// The order is the whole safety argument. The backup lands FIRST, through SQLite's own backup API, +/// so the rollback target exists before anything is touched. The rewrite is then one transaction: +/// a store half-converted is a store where some ops are v1 and some v2 with nothing recording +/// which, and there is no reader that can sort that out afterwards. +/// +/// Projections are dropped rather than converted. They are a cache over the log, and re-folding is +/// cheap next to getting it wrong. +let upgrade () : Task> = + task { + let from = storedFormat () + let target = BaseFormat.currentVersion + + if from = target then + return Error $"this store is already format {target}; nothing to do" + elif from > target then + // `Releases.runPending` refuses this at open, so reaching here means someone called directly. + return + Error + $"this store is format {from} and this build writes {target}. A store from a NEWER \ + format cannot be read by trying harder; upgrade the binary instead." + else + + let backup = backupPathFor target + + match Sqlite.Backup.toFile backup with + | Error e -> return Error $"could not back the store up to {backup}: {e}" + | Ok() -> + + // Read the whole log first, outside the write transaction: the decode is the part that can + // throw, and it must not do so with the rewrite half-applied. + let! rows = + Sql.query "SELECT id, op_blob FROM package_ops ORDER BY rowid" + |> Sql.executeAsync (fun read -> (read.uuid "id", read.bytes "op_blob")) + + let rewrites = ResizeArray() + let mutable unreadable = 0 + let mutable identityMoved : Option = None + + for (id, blob) in rows do + match BS.PT.PackageOp.tryDeserialize id blob with + | None -> unreadable <- unreadable + 1 + | Some op -> + // The line between the two cases, checked per op rather than assumed. An id derived from + // the decoded op must still be the id the store filed it under; if it is not, the hashing + // moved and this is not the migration that store needs. + if Hashing.computeOpRowId op <> id then + if identityMoved = None then identityMoved <- Some id + else + rewrites.Add(id, BS.PT.PackageOp.serialize id op) + + match identityMoved with + | Some id -> + return + Error + $"op {id} re-derives a different id than the store filed it under, so this store's ids \ + were minted by a different hashing than this build uses. That is an identity-changing \ + migration, which re-mints the whole log; this only rewrites blobs. The store is \ + untouched and a copy is at {backup}." + | None -> + + // `executeTransactionSync`, not hand-written BEGIN/COMMIT around separate calls: connections + // are POOLED, so a `BEGIN` and the statements after it are not guaranteed to be on the + // same one, and the transaction would silently cover nothing. + // + // The stamp goes in the SAME transaction as the bytes it describes. Outside it, a crash + // between the two leaves a store whose blobs and whose claim about them disagree, which + // is worse than either failure alone. + let statements = + [ ("CREATE TABLE IF NOT EXISTS store_meta \ + (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + [ [] ]) + ("INSERT OR REPLACE INTO store_meta (key, value) VALUES ('format', @v)", + [ [ "v", Sql.string (string target) ] ]) ] + // Only when there is something to rewrite. An empty log is a real state (a store whose + // ops have all been cut away) and a statement with no parameter sets is not worth + // asking the driver to reason about. + @ (if rewrites.Count = 0 then + [] + else + [ ("UPDATE package_ops SET op_blob = @blob WHERE id = @id", + rewrites + |> Seq.map (fun (id, blob) -> + [ "blob", Sql.bytes blob; "id", Sql.uuid id ]) + |> List.ofSeq) ]) + + let mutable failure : Option = None + + try + statements |> Sql.executeTransactionSync |> ignore> + with e -> + failure <- Some e.Message + + match failure with + | Some why -> + return Error $"the rewrite failed and nothing was written: {why}" + | None -> + + // Outside the transaction: re-folding runs the playback path, which opens its own connections. + let! _ = Seed.rebuildProjections () + + return + Ok + { from = from + to_ = target + rewritten = rewrites.Count + unreadable = unreadable + backup = backup } + } + + +/// Put back the copy `upgrade` made on its way to . +/// +/// Contents, not the file, through the same backup API: connections already open keep working and +/// see the restored data. Anything already read into memory is still the NEW store, so the caller +/// has to say to restart -- the same caveat `LocalStore.restoreFrom` carries. +let rollback (target : uint32) : Task> = + task { + let backup = backupPathFor target + + if not (System.IO.File.Exists backup) then + return + Error + $"no pre-v{target} copy at {backup}. A rollback can only undo an upgrade this store \ + actually ran." + else + match Sqlite.Backup.fromFile backup with + | Error e -> return Error $"could not restore {backup}: {e}" + | Ok() -> return Ok backup + } + + +// --------------------- +// The identity-changing case, and what it needs +// --------------------- +// +// NOT BUILT. Written down because the shape is settled and the cost is not, and because the next +// person to need it should not have to rediscover why it is bigger than it looks. +// +// It is reached by a change under `LibSerialization/Hashing/`, which moves every derived id at +// once: op ids (`Hashing.computeOpRowId`), item hashes (the content address an `AddFn` carries), +// and commit hashes (derived over message, author, stamp, PARENT and the sorted ids of the ops the +// commit names). So: +// +// 1. walk the log in order, decoding each op with the old reader +// 2. rewrite every hash REFERENCE inside it through the remap built so far -- a `SetName` points +// at an item hash, a `Decision` at the versions it pins +// 3. re-encode, re-derive the id, record `old -> new` +// 4. rebuild `commits` PARENT FIRST, since a commit's id depends on its parent's; every commit +// downstream of the first change moves even if its own contents did not +// 5. rewrite every table that stores an id or a hash as a foreign key: `op_owners`, `op_branches`, +// `sync_pushed`, `seed_ops`, `commits.parent`, `package_ops.commit_hash` +// 6. drop the projections and re-fold +// +// Two things make it worth the care rather than the speed: +// +// - it is DETERMINISTIC by construction, and that is the property that matters. Ids are content +// hashes, so two machines re-minting the same log independently arrive at the same ids, and a +// push after the migration dedups to nothing. Test it by re-minting two copies and diffing every +// id, which is what 8.F.f asks for. +// - it fails QUIETLY if any of step 5 is missed. A dangling `op_branches` row does not error; a +// branch just silently loses part of its own frontier. `SCM.StoreHealth` already reports exactly +// that class, so it is the check to run after, not a new one to write. diff --git a/backend/src/LibSerialization/Binary/BaseFormat.fs b/backend/src/LibSerialization/Binary/BaseFormat.fs index 38cef2b554..e8207c922f 100644 --- a/backend/src/LibSerialization/Binary/BaseFormat.fs +++ b/backend/src/LibSerialization/Binary/BaseFormat.fs @@ -3,19 +3,34 @@ module LibSerialization.Binary.BaseFormat open System -/// v1 is the format the op-log substrate ships with; nothing older ever existed in -/// the wild (pre-v1 stores were rebuilt from `.dark` source each build). Bump on every -/// wire-layout change, keeping a readV1 beside the new writer: from here, stores -/// cannot be rebuilt from text. -[] -let CurrentVersion = 1u +/// The format this build WRITES. v1 is the format the op-log substrate ships with; nothing older +/// ever existed in the wild (pre-v1 stores were rebuilt from `.dark` source each build). Bump on +/// every wire-layout change, keeping a readV1 beside the new writer: from here, stores cannot be +/// rebuilt from text. +/// +/// `DARK_FORMAT_VERSION` overrides it, and exists for one reason: there is only one format so far, +/// so the store migrator has nothing to migrate between and no way to be exercised. With it set, +/// this build writes the version named and reads every version up to it -- a SYNTHETIC bump whose +/// layout happens to be identical, which makes a migration across it a pure blob rewrite. That is +/// the format-only case, and having it mechanised and tested before the first real bump is the +/// point. Never set in production; `LibDB.StoreUpgrade` is what uses it. +/// +/// A plain `let` rather than a `[]` for that reason. Nothing pattern-matches on it. +let currentVersion : uint32 = + match System.Environment.GetEnvironmentVariable "DARK_FORMAT_VERSION" with + | null + | "" -> 1u + | s -> + match System.UInt32.TryParse s with + | true, n when n >= 1u -> n + | _ -> 1u /// Binary file header structure (8 bytes) type BinaryHeader = { // The blob's format version. Passed to version-dispatched readers (makeDeserializerV) so a new // binary can decode an OLD layout by branching on it: the keystone of any future format - // migration. Bump `CurrentVersion` on the next wire-layout change and add the matching readVN. + // migration. Bump `currentVersion` on the next wire-layout change and add the matching readVN. Version : uint32 // 4 bytes - format version DataLength : uint32 } // 4 bytes - payload size @@ -43,8 +58,17 @@ module Varint = module Validation = let validateVersion (version : uint32) = - // Reject formats from other versions rather than guessing how to parse them. - if version <> CurrentVersion then + // OLDER is fine, NEWER is not, and the asymmetry is the whole point of a versioned header. + // + // A blob from an older format can be read: every historical reader stays in the binary, and + // `makeDeserializerV` hands the version to the reader so it can branch. That is how a store + // moves forward without being rebuilt from text, which after the flip is the only way it can + // move at all. + // + // A blob from a NEWER format cannot be read by trying harder -- the layout is one this binary + // has never seen -- so it is refused rather than guessed at. `LibDB.Releases` refuses the whole + // STORE for the same reason, before anything gets as far as a blob. + if version = 0u || version > currentVersion then raise (BinaryFormatException(UnsupportedVersion version)) let validateDataLength (expected : uint32) (actual : uint32) = diff --git a/backend/src/LibSerialization/Binary/Serialization.fs b/backend/src/LibSerialization/Binary/Serialization.fs index b582bf64a9..861da616b8 100644 --- a/backend/src/LibSerialization/Binary/Serialization.fs +++ b/backend/src/LibSerialization/Binary/Serialization.fs @@ -58,7 +58,7 @@ let makeSerializer<'T, 'ID> use finalWriter = new BinaryWriter(finalStream) let header = - { Version = CurrentVersion; DataLength = uint32 payloadBytes.Length } + { Version = currentVersion; DataLength = uint32 payloadBytes.Length } Header.write finalWriter header finalWriter.Write(payloadBytes) @@ -72,7 +72,7 @@ let makeSerializer<'T, 'ID> /// every historical readVN alongside one current writer; that is what lets a new binary decode an OLD /// blob. /// -/// No reader dispatches on the version yet, because `CurrentVersion` is still 1: v1 is the first +/// No reader dispatches on the version yet, because `currentVersion` is still 1: v1 is the first /// format whose blobs outlive the binary that wrote them, since before it every store was rebuilt /// from `.dark` on each build. This exists so the first layout change has somewhere to go. let makeDeserializerV<'T, 'ID> diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin b/backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin new file mode 100644 index 0000000000000000000000000000000000000000..a89b241afeb51d2b5c71924b91a8fd6f2aabdab0 GIT binary patch literal 106 zcmZQ(U|>)JVn!gA0%8uA#G>q+#JqGSw&0SKoXjK^pUmPC=G?>rF{mm&AkFGq>0gx2 nz?PAkQ;=GOte+($wYY?d%@0VXrm*?urX;TV literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin b/backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin new file mode 100644 index 0000000000000000000000000000000000000000..3cf2323cc577ed13bad81342511a14cb9eb2e1f2 GIT binary patch literal 5175 zcmc&&&2!sC6yKF4$BCVO(KMx>8cL@FG#!0eb~->CXnSdy4s8a8-egNoG_fU*B)9Rv zP);!1IB?>?KfoWrk>L;E#GUC4Zk#!VU3=eKJCf~mdZ;IgR;%Cp?c3jbdVcbGLdb9I zpzQdS9fbb!$G@kX&-QQ9TMRrJd_>pD%38HXm6EEJmnzHWR*#YOYL!-x zuU1b~PalPq&?Dp4)GSD|GgVThG$dqnb&C1Vx3DrkU-*)5g@0ghliy{Z6_}tWsyHO& zG)uy0=NQba!hld)F`XJ*(l$aU%$3H(M1cvl1f!A)Bb8_{ERiw{Rc5*b7o4X87s`gY z3}b3u!C@VTmrToc>TaXCX%vk{s;t$`TIc9)DDxN;h0eTv^N>9BcZ>P3* zy+kI2mq%!wZtP{4LX^6+o=4FLQ>ox)coMu=gTZAOFzHsqD07|d`0*%hM_RtFXAl)H zi;xm@{UUl?MHB=cC%{aC^I3vmMXW(jAvFx>kW5fu5b5wOKyX_+fteV-0T^kwPltul zS4Prr`Mz)kEMXrs=j<7PlfcvnQFfQI(YgMDG%RC}?1ZxFS!Fpk7V01sgxtEji~ za8D)@-6zter>HBMp?ku+Z)ACr(nCVYy3{5T*Tx5m%1MHmOJ%9WM+M85`qb;1#nYCg zor2qi)Cdt@$7G^Diz%6$)72SGQ44;5vLfIONGp_-BacHY2(43>ph%-jENI9@%;K4a0OyvuQi7;kcWYZ8mJvZ8mJfZkY9E(=m;D!)oN2mPU?SO>9%I*^Xf~E!$-* z#-`nLT*hhJ4MyB>7_Hm1Ow*|sv_c~dTnCHaxa%u&e>u%qsqO_)rbBr1Prm5%W$A+) zV{a}R%V)3mnqF5_KS&ET9d;Ck29UrozRJr3NdtEQuQOb`qzZLXW zhN7kJmgBGLAfXBJhtE;O8w$&sv7%ZWx;zMswS;ZW;#ryI~l{ zlWzt4*YnwFg-LdgH|uN9_};jRquyN)Q@s->ubw$mL7k(Evb4P$e9V63IrD5ch?pbH z0sf=_+w#zpPj3UZ_W<%@?Ox-^j;VR zNB9-P612QkXSCLAT0)mQVdS&leKs0AGkHNgKen=`AJ}p~jH19-$VCBL8^fGw81wEB zpne#j*n)vS5-Ts%tGB*?_~*%=Y9;NT>o$e_F*gizdz3toKU`$)AJ6KX86hI?@^;(8 g?m?lQGVJ{P)z$2;+dtla@%;DyR{lQZ&}`oS0ZL7&mH+?% literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin new file mode 100644 index 0000000000000000000000000000000000000000..041cb1aeebc5366927d993f9d61704f61f0c7735 GIT binary patch literal 3100 zcmc&$OLEgd5bc&F+llNXgnS_UQf#PVWqm9u%Pe3^6;SLI$p%$q$tBAvyI8;xxCdw9 zEF2+7wz@|_V-i-;M(xq-*FCR$+JgXa&;!8Y-7DWefBoLKKP`T2dhdNLkmaM00c0qP zfo##J0Zc#{#HvAT;2+vL5^zlZqAX0duh-AFCQ7|X(|DWG!`lFoIIUTp=|rGF*FP;5 zqU19k&|>S#&m_db7eDI}2K^>N8^@`i_Zfl?4x1=Ub_gN)7(Jkmhh)4UqmZs2CAnjN zOh2qrUTKNPuzjfaP-&@JyVKn|>F>a?uA{!&*Z1^;0&{Bv0NxpQsOFMWU5ZA+1&EZc zSXXxA^s4N>i}56+r-l-9DFA^B_&}jp1nyiEqn3{fk1zGgj}7^kVo6KnF4Y2Xhz_twtDij3m;-%bUepQ|fDr0GylV0a zf6h-7MAQaqfg~eh%)td8XH0z~@+L`|Ox>xU<&huHxD_O=-_GaLcr<&;8j?Yg{t=oW zlF%1Xl1|T)kmoekHqHBzwFZ`J3_Z^_21D0$n|tf55;wgqrn2mT>6w;Ounf*D&vpt<%W?~1*Dh!s&om5s zD7gx)G;nMle$};4lK$2#FGlrbl;o!rwt4c)3-9V72=y4EuND35Pwwfeq=ute0QVKV Nys#l^L7WDce*hX#p*R2l literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin new file mode 100644 index 0000000000000000000000000000000000000000..c8857a8cd44159612b9f114ba4c3754316a1958b GIT binary patch literal 198 zcmZQ%U|`q>#Ed|^4MeX#`(DT-tYQK#nvpS)i8VQ~IMorT7|d{D6ldTP<>p~zVrF4w uW9Q(M;^kuy=jLYQ=I7-W;1(gqR0c+()rqo{q!yQmaWil;GG;O`fO!BaW;@6L literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin new file mode 100644 index 0000000000000000000000000000000000000000..60e6e38de98ca1cd578ba0b52787aab000b9626e GIT binary patch literal 281 zcmZwA%MOAt6a>(@?G?obk-c%_nuH4Y;CF~2Zd^!r{=5`2UC}f*XC{3JfLwraBMk$m zUzlML?bHKX7INH0A{qSm{pm9{2Le(VSMM36i${c=oj-zMp_Z~rDORLKLh5*R%@E$% bFqp!nw75%T6M1&ms%~!g$Fp3L`q$w9+=UkF literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin new file mode 100644 index 0000000000000000000000000000000000000000..326d8c4b0dd682c1bafc10eca21724da5e248534 GIT binary patch literal 39487 zcmeHwdt4jWoiBKt#Ca+2SJO5V*$$8xn2`hs8xv#fBu*SBwVgJpacv=lAqI&ch;h?2 z5}22tLOg6RFTV|bfgiE4!A?b!*0tow-McY#yIWgsleS40@7;Spd-t~8-0yERfQ}>v z?0oL$<3CpL8_oHh-~0T|nKNfbGygd$BI1FFh=@qQoq#D{%O3`b3hBcjiG@B4(vJ#4 zhNXmJ>BF%9S2*nxHwo;|lTbz`*h7&ZniG!l94TYiLG@7x5*~#jq5Xq!GDDG|vW2A# zQACs>NN5j5%AtFnF`iiQVUeU6pE=NnnlVoZ8u3N zYY)f%R0Jc_uZU-ne)b@w-_GBC9fti1i}bT!`TA}Mr+p6aL}bEu?|aBzc>8HuFqZNu6btGIZwT!i!u~8+f!`my&0VIPm3xAd=1M~21T8HC;VsJip-j{AeK zehH6+#Xb=h`<*#6!l)GFopakiKg6qpHKdD}U|tm-a%~F+a7hX`g>b z(ph=pdRyg5QI=0bt||&<1uB;$NSIz}*LenC_^0UxrB)I#C0wk(c;sT=#qNte{HL~0 z?HA8o?74XQV$Y{#7ke)rLIuh^c2DpZ&tU(URD1X(S;QO?ks72|E)FY|i=_N|+ooP$ z^9KCGtuy|wbN9%yCX=j54^Iv}%&G49xVQh<$E9oN$V4`)2=2Lf;^LW4ZLs_J#ctM{ zl$@gcywsE&lOZcLCEov)X~|j(l$ctSTa;%4Rz#H_884Bu!uyZ)clS5<@95w8%hQ2< zL=~}EHsWP6;-&ZAY{9<7cy z6&d3TK)fbP9TQgw6Qg7J_umgZw5q=Q-gLg8FxOnfN0mhJ&+>^nY(zn$d-ho^KmC2y z9<~(N4^>lsbL2M_gJ*xUjiq5rPssjI#8vlQ>1w{xz3ECgIfp&P7Kxz?Dy{vt{W~5bPse7Zs4!%*+q~uyz_^Ot*kw4 ziS4~hb4>g@#ks~Q(q}cFS!h{f=Cdt#jDAYu;^O#ab4~aZEKiV)j3pzVn_FDOn=NZV z@Z>U$ke+QR|H?$R-(4P)v!EYhJ|{<>XptNXU0he5T1{=5x)wfj3(64TwV_Z^+BHz!#0c zuQeG8t&+d|4@@MYsBkSwFz{y68W?2EG3UNRrvAXhXIacqMf_?6hhLLpB3h{7^G#N( zVU>wrlbe?ZKeG7Z0@4PH3i&J;1phM;(lsVNhrG?juBC9bHI9Eh&t$NI&#>AAo*6`u zpKG-eXB_Jd0ZyFpGZ|piVIAI{9MbLJpRo$ z-(m|cNL#*F|N)mE;P^Jmsqlk)=-8oDKHsx4SBiNB65ZpC~&Gr za*B!yX2!>_S+gb%CdIvDj58YI15X)L4=+X%nym}WONy;lQ%RiB5?8$XNm&TBlk6*GBnsdpeSBtHt;(WI9<=lMq*7UsGB4nt=^t3FPh*Xa)%w3fO z`_awdQ?VJT&cALbTs=)D`Zf`L>vfYEZduGG(B$VAn{$gGFQ>TBm_sokSLqx{_2d#G zB;;5Mk<%}l%%(!LQ+L6>;1to=69&i(L*GuZ=k z+Dw*U(HjdXk;&dE#BqE2!q?JYnZ^>Oz40cCczy9A7LoqiA~pzYiS5tNJ@;JN(w77_ z_fk?)`03W@V)N{%>DH(iel|b7c%E!o3RyN2huQb|6|?yF`P3E5rbSPSj)_?gOp&G7 zn8U{xl zik7ZrE=P$w|J1CxM1m9d`SsL%eR&+o&PGY&X&U!FuCJXm#3ycO6oHp zoRtz!yR4aPi5cJ&ztwpB%Kt6T=rGr|ux}X>no+#=j7CPYPWksoDSXbB*B{=CI zSt|9M_m){DHM}~w>rr=8FMpqPXj*hQvKU?#N|8}Lx-uNiO6nn`nC8~1XX3L=@5Y;p z^U$NiQS-BSOo=Q(nfAt#`HR^BE;bgP7qAnL*jPEa*o=;i4P1g#MaHsmVM{FO3br_% zE%NM$6g7*V|K|L3-deEY)g>=j3g;V)IZF+hc_uz;-s0C@=a;65)_+&KAMJi)xwYTg3CY2sr^_ailJ^5*3?D;;djA)?K#5 zwp2=dk*TBzlMzE|iq%+{TTqm0$TJldMQbz(F)8sfimV||XxviWFHcVl@zbYk1^lyq zrWE8Ea5@F4sqCDiM;1WGQ&r-y0dFo@;*aKpj7XAY#FLEJ(Wc;^nJVgL@vinTZ6YQk z2h^!A$*gmTH5F$8d3-E7tpBuMJtB$(WC%>NW~eoyJ_8=8XGQU;yrsA(D5nE6;+d%> z!n|3*c_AiAHjKlU7Rr}65x(S5z66O+Bm4X;`5ZmsBV{gP#fMm_lej8nt`CUo{ShvB zmY7Bx5u2pMkv%Q#(I!bld&#sX&_;)ic7!7_ooW-Pg6O1LjUIcNBy6;V(~@K@iIQ-d z#n8H^0DMZiZCUJ!3&D)g=)}}o+0PB+XH}q|@Nw2NQ2mwc=EwLnw5(~5oyQMmbS|kdcfQ^=9B+#CY_N0x3(}}ZEDQ{@ZpaYVM zm!fYm=UZ?HqLF&GdU)6q1q^$s)G3LY&dXHON7@RbL>LwUy;7YTbQ~J)NK?n)(GyR) ztY#LeNgh!X(({*KeaXBkF1yf@pPpkVOt)m2q7#G}I{l1t;IT!d6g-gyBd-pk*CUBQek%L^Rh%bVyF^^V>cW*K3>g7sb@>} zh6#j-AciE`aQM|FEBTcQQ4~g^n5qV4>JMd07Lz5f566+TqEkLI6+;h3JPb(~j4+O1 zAj3c-qC(X8TWV&AfnsuUOe&VIDD0FX)agW$9zX2o?9s4iln6rTR7FhIqbp5-S*r=h zoHQ|JF)VZ91ewKfniD3-9M)&u#F&NTaGJGX4xUKnuUckUzASV3GUIZj`?9R%vrCql zmJfH&W!cN+AyQ1%fSYDM)K)a=sp3Z4Dx*Zy4)cW5EGqMAlz0VYUQvNppRCL)DDhHy zjqR_Hti(HZc4d_4k^*u&OtAE0XBYK^ahS)>F0_$24)fUA1!m1S%#>Z>yJH3>N8>P4 za*f3-YR6%w>>87Kyns=9jm0b^D{D@o{0hTN?IayMhDYl#r&?mt7~>bBl{`N47#Rj; z-T2I7WEkb<_{@}Hq5XogYkXcxuyDK+(M$Oij#sN0UvK!ai$<;SdBcxg;GIAdhabBp z6z9Y-a|?%6nwTi%R^phsHGx)295c5j&`gPxTeR#=hlywCFfmO!OoZtY{t2p>ND0NB z+pAYbjm+{gO42nMC22zF93pXawilEbEWaikuXc2f!zLBWE6fd}S2M~xNA{Fts1jM; z`6Ah!QKFw8Mus>+8Sq_CG7`hc2$-iS+A%Cc6DlJidDIT2IB3K$y!uhz3F!~h4hcyU zl<`7F$y|aku}nyupp4--glSWpP>r1KOvo2)n0N%_qoNGim$Y!c1mxv-eL|lQ#)gUJ zrW}j3eE!Q4y=6Gz<*=4M(LA9Mk^snj=wv@o^@3|i+ zj)1&WjODygUd;saQ5&XDYXb6-U0GVG#$;=FX{e-x7Q8Y_G$W_RVa!O130j;AFCuC9 z3^|O5#0iNIl&naYn22-*D{$d%0;>o6@04}q{yqb>QQ|-l}HP-q$uLFh0o2J zt?q1fbbVsX~<&BSWM1_5W--%P9^gtHbW z-#0E6=g5IGgwYn1uQuaikD}ORawbsIslGRo2^l5xX^;+{d4>^>Q}>&)I&eZ6Ml8)vKIU{POc0cNA6=uO zldYF&%y=na%n$-6R3Tzvu9RO(IkK~b>}>c43p3|x2$wOP3jj@f#GawKlD z?}>`z1zpL!nM`A@t%O8?E^#VI9XHeTibvNZ<-Rv&e8s>ercj%2vd=M#fT2>A``p;J zDrQ*HP4+cG@dPL7Ci|M8oMFm+ZR`jrj_OIueQf*;Q_cnDzBaye#hy%8c)wBaP@D_V zC13^VW5-+Zh(y^Nh#5^{w#r0>xM=9{ruP&{qI)n_HC-b9jXQWFm>Mi7bY%Ft0WE=y>C#Qs3;CSjjUsD>kZ?qFjr9wy4H<3 zsZkV%K6PW_5){Ru6Wy4&L`8AvJvSy!Id@XvR{~)1gnVj(x9};zn;wGH=BTG|?;wGB;g5tQZlQVz(q)=={T*seCKZHXWB?;;B zlN>^iLb#+qVKIU*wwQ@!`e=CR*10g&$eBJ)Yy!n3q=ykBXZTz>C1+Af>Vs1fx0NbK zSWayA*KJWNabq7@Kh_U6v;i0P!?J4A$7Bt-o=2t_W=Dl2g+>>{vZ=$c z1zeVvb4jC(VShe5awn_*epiiV)GoecHv0p$(c4~S;(){iaTH69-X-U(OdXJTpjJ} zo{%;8p9B>(%!`IBnjOVf4*#TCM7!W8R_vos=*M1+D`LQHL+t%?pc^I)_?@@tA4x_! zu~fzWz<8*)*{@=#w^epP69c5&?}63G-cXcP(NV|^DMtdMCNqjLqq)FxsTH_{051Bq z6OCE^C$PEN$JCV^z!dcBTRxumdxS?!jO_U>#N9ptwQL|pf;1dQKTE*PX6!Q@f|@4a zhBEdcIKjyqaMu`17@W5OZ5?#yxb@vtBiixo&lFMqb$GyWgY9giH!`6e{68`Dd#8!v z2q2;04^(b(h?-lPHUrCorla z-jnOBRQHb0dBo@3<#V?4KIcK7v&-k)?Q^#JoO@Aj@i`CqoSi=BQJ=E|2?o13u?rltI%C8K6gDRNy|Jv)Si7;d34bx=kY6*(R&sO6uz(ueUUn0crr7d2E^h ztpF!rBcP6u8vxq@+W_r=t%TenBX&@s7C>wbfJQ(wA&I;VuoR~`VB-OVA_5<1oxm8AVPyr4|Y%sHa z2VgfLiF_XbcGvGB#2&ziCfHk#K-3=qbW$yBmRPohm{<$twcUWjpsPI%I14xmI0!gS zNJyzY12}cPrOt-$lY~4XBkWW-1|YT%0X=|TLV~>Z9N;kE3?cdeBbrX2&?@t`HMzpxjvvr~sW4emUW_6HVY;2iQnRNO2;PP6q%@;oL&VN*U2e1+;|3hBkGg zDV>xb5;>SlXlw`a(9s3hM@Wd45ZzQj zvPf)5V@C(zAR&ny>EJ+uIt~y5O*yIwiRnNv9LN#JQL069N-XW{fHKs3e5kK#sIQzK z>Z=**s~_sC9_lL_>f3;F=}@0@sBgBedFHvq9>T?YB)uIfV2FL(C z3Uwm_HxBjLhx(d^`Wk^=Cy`Yujdc&ogALKBrU=4?#!nii#a@uH{e=A}@;2pj`K(OU$mU>vE*3__+M52qQzf&=e#2iMKhq-TmTM&^L548@*vJS zf8-Y}zpCU5Ea#6{%|L<=`~|;g!IOu6RS85MFrSnW{rNg8wpoAz(%NGtEx)QfUkBo1 z%lSHDfCWcTDREj!0ayOJ$0V)tEX)4t?pJ& zmHUi)>(x^q0=wCL#N7i_?ThX+uelGQ`tVbjAkZA)F>F+IliYh>avyZ>M$K+di^ zyPMs6+%4`io=xsHw|$MfKF8hQKH{mhxLZ7p?gohLb|2<{UF|;YZiMbOPt_0HC*8g7 zLk3_vJ)3@h#8c&|cOL@r5qEQu`$NX?h`rg z6YhhaO7{UzW3Ic$Q|ms4zr)aVz+G?lv}L)EyE|c61MltvPlu-x(vNytJ>^Rwv={W9 zo}D?KE!a8iK4XGIEikN{hhIJJmLGt)71q~*t;JJrAUaPSYI@zf^4!O<+XcclPu5vWb>J&;wE<38%PnzQ|MWX*Z#|`^WBWu&Wuu8VjKCG#uRonN23dijY;i8_k|3YPK`*u?xH9h^41G7x8O-*S*tIdG!?1z|$6s z7CYqbK(Olaq5P2hKrRfcgh6}F?ya82tEbT3Jeuq%(5=5X0KG>Lm!pW?8Fx!IkN9`t zZ!63?gvQ)vgf*Mse=D{fZm{q1G#R8OYjZbxnm^eAY>#KN)xA3oI$Dr0KXmT^b2EJC zLYS)E%?8?q{DYnyd60Afxd2(KK-Pvxc6*xO(LqnS1?Dx!x!dlOTjdUFm35QcJJ8z5 zfCaE=8;of&qe+k#?;-^bSXN5{8$DIIh-D*8?m@s>&5(Tt^}F1sSHZO^s*w!XjKnHJ z%=hL%=TVb;r+YUKV-A9-A=`Z%4%fSP_3!sITs_@`?$MFwK?`}Ba?uz~(A{oEc@ylU z#67&qvx#@_x1grevm+bE?Q$RF;rcfC+Xi)qjlZsjt^?qyg~bh0lc7y^q4nzecfg-s zv;q7^I=S~)5s4}!OM`naI?c{F_jc@}&m&%4RdPsd6q3JBy4uqP!qe`0i>H-$pUg!J zs8e-%8qH{ft)BXK5!q%bT5T5hhe+5icl&C%)r6R!^CC#yaNONcgnX-qE4vV_PETu& z)Wwd%fMZ}jYJ$vNVBHJp2;5HCc?OKza$x^v_}7fuqn>i)D-0mQ-S1L&H$lZANI)XR z=DT<1ddki4vI!l>(_V<`R%pa%Mk3o0 zw>BDIDy95s@l>0TTzeoAf;~;JvH2H0WT|H}yl(*6?josacYA6f>5yj|ncC^@HhbDv zVKk`b(V?q7h z-Q~0IrTfQy_I;1bczU#(O65nkOJBAFK|Hn&KfWUve?X>xOqNAYj#J(1vvuO%U)8B( z8oSbWwQ2j+CfkkwH?s?El9+W@n`*B%Ij%NsxZ32r+Jqtm*GN01($1EvP32dcnw6}h zU$-H*<%h)%?89T7%Ldo+gZ9Doyi_V5I5=27Sj7+Y40Q0(2{B*F4A&?MS0YijWw2s! zGiXb(Jr3v?+`eCx@O?C#Off`j$!2$ z5MDEI3fmE4r=(}!z+O<553^JaZuYYcmh*$_fjuleoO<(MjpPIrV|U;rRPQI@vWQMH zlstA092(e1CJ$E01Vj!$1~$^;^C&+4$D|IOJROYaV;>}Nx$d)5mm5Fpy4;EHI!X`H zfJbB7B^{l^_S6g<8mt;zw^kPa4n@;)x#qL3&w4NK|LnwN$K{UCj$gK4ZoJ$%g9rYU zggjvv@F(8Ui{4pxAu&F?+Y?i=VP*`2_jfTaEWU>SHT zK)(ybHAq1e*TV#ApYJKn&_Cc&eU(WEtp!=5D`XQFi?_90xCxjI_w}}Io0L+R zJ^Z_lWUPq3R<t<+OTq&}>AAe#5=aCPyX=3g~Pdok<^q_6Ivug-?;wX)68 zyz4Z-0vqhdJS$6~Zmr^@ht5`CIlAYI?d!1}>a9j`=*&6ZS9iu&XU8`1wM(j}qJ8y8 zTwT5fCm;pZ`&FE(`xA}CbXV1kUXTp zL^WgRY}-)Z9$)jhYh`DB^+%(J`gZuX*86Pdu*H6iuf7|VAORtJH3ABwhtBQtRoSqO z@f*Qj$XKyr=!4-WO&hWhqLyN(lcpUqd*!KR>W1&?Z8wu7BB zT`TLm(%XCGGzM@_Grq3$_93Wzc(hU`&?J+BF`4X|^vknB<)GJyu#(uTB7fYm|5{ng zHG65=pH6Q5*xBDP)VuRfwoP~+Nt>(l=jHv~zdu!a&0)LVZnLps*wVks27OKjU5mW# zaG0YJROQwzY*;c#?cchq2w(^0dIN{R+;0YL?z96&N?zX=d%B;jfS6|Dbki5_1= z%|n@|`u6qq=FNca>+LN)*W0^m0jI9FA8okaeryYSsq3qzQ(v~)zU->|OGW#adycbL zD8ATz_VeblFZR{^;n1l+HSYXlXDfETXs`RC(!UjD0^?O$%#{pH?+{FmF>zwGMxbC>-u_1oAk z{Z;9~zcg(G^Un{G*R@Qs1tm0bfF50o5`s0p`*X$dv**mY<*I|bx z|Dvwz_oohBv)5lMt-WSHfQ-FncV4sa{rxF$)PsX<0=C2skMyW?9?ut_y%NuHjKbxe zr)0xufK@#*Qth*~fWT*KRw8MZNLnUF!k*Zvqv#JGHG6dMw7 zy<5BvZ;jXKt@YM<>%9%$t=?_kMsJgMySLff;%)V|dE31^ygR+Syt}=7ynDU-y!*W! z-cIiU??LY&?_uu|ZX)RC=lG z(z;9Kmntq*UaGoOeQEur4VN}v+H`62r7i67!B@WBSKjO^Z}FA4&XC=py9iY3l|K7A zU+F=gt;c6OJ#^4MbgPFKdn9n3y0E>!(zkW}&$}-) z_m{ekT-fG1$7*%$@U7eF+U45mt7>)abA9Nu?R0gzPWx=NzUp>%9`JFkud2p%+;!Mz zYj^FD3_SIz4Tl$>Z5tgDY5b)FmFkK9J^gLIs?txKu8s?hzN(4~O)kV?Z~tDF>|%$n zx~hMlud2ycb>Kp?>x8d*x379n|4BA~*b;kC89H~Ctrl_j0T~j5Z`Iz(u0sR10sBDd zK-s{$f%1Xoe`@*dmaFZXeKx#Da3fBBd^YSO##o6w;IqNVLq6MKpS|5@-{G_G^x1d$ z?7OiRigi$*eZSA%;j?%8><4`IgFgEqpB7jFHuGf_IUEhAR9k2ti z6M&1J<+yrTejI?y*5$a$U49aPtJUSW+FgG7-`j0=Ks#UuU?*S~U^ie7U@u@FU_YP( z&s%GC zN>`Pu+O^)b!L`w~$+g+F#pQ6-xSXz9SDmZg)!^Fd+U9C>HMzFCnq4ifR#zKqGkc#= z&b6_=etjRz_PJp+0%3II*1=V^K5j?%Ac|` zy_D4FXR6a+o0ZF!o|XejCkv{_G;w%0iIAk#C+P8|!9)E` zv06dT&IN_!pt7EQ+YDYHAPB7WViZsWGeT;3UEkmA$@i{`Pbx#H9%i_#VaxSX0U66!i*V0Fh&T<1qhA{k`foh6BidO1=-KiJO?3c(4GsT zhz+5L71%K<+#B8FV@38_XskxmCF%6qM4cXWY(pTHEJBPI*o=-9=RYTC*f0Vo1ig9? zs1QuNCP>sYyr(TFma&9l!w@U7+9U(m*pdDqyXOZZ;)9Hqln04{ZZ;c4wy5}aJ=W@s z43;&6UFgCu2#62W2<8|Os}7P1I)ZXVmp(ey&L7*-kHDY=lv5D{x*u{q0> z&5nQ)D%WhvVvo~-#(u=oVywX@E{ImfeiS&SOrgub^a)fw>4U&F zp8WL(fuXql>#x85Km><0ttZYXF-Z`0c&Cn5mni5GlQmi~Nh>DmleB_1Nleh|b$F*v zk|rro1^0EJirc>aI#9(OQ1Nx3m3IEc_ zB+oj0RnDLUOX0nWRt9Wa<+V zq&0R(V$UFR%|^VKIX5#8?^~w#7q2l`=?&4VO=k9N9n&*?tbi9j>Xzd9cv^9$PW)kD zpj3B=cw>@IpAeW2swuHDHZbO@NFh%4jLs`n(~`21O$o3zHd9E{$LcVS#u_w6W2_c2 zHDv0tM6D??u(Ll9IVExmT`{1~&j1vp9>BwMvWx~nmyju%G=^-Qn5D;*B~zQANfa|R z2?m`ealHC(EFglLOfN`(Ad-u?gI=PJ7t%ct`Cz0f61|fvF3eLsOfN~Z;x(aJc-L+o z-id>E{^mEyPB`)X$Q*+7^0J7qH|n74qL5n*gpMV5RaTlu?N!%8kPGVQ$e zL>L*_n`Kw)ZVgu>B5#{USu4~v zG13SoLqevIrAam=iU}sssLRI2sL9MW8Z=2lQj#$%Nt0>BY5g}A5Q(v#W@QmAvR(DJ zj$KPE`KHJzG|i$GKjk}wSyX6iLclwRHa}IKjK7nUJ_eNR+7lU!aa2prJskvw_gNt5; zlVcjESKu2phi>Dlx!1Wx+>6}3++Eza0Jt>*fA?|=xRi11 zzvR*+aLnV8{u;J(ZCq4pebUvo{|4sH@Ri~ByjImE?q^_-LYN7(jD?n~}N?!UOdaZ|YpE}vV@ zea5A5-{USpODDJ<;A*(l!2g~5PuTTsgkR77hC3^Hxf}ZnxF2yVxdC|9$^8>_@8jaR z|BF8#_s@uk5pmnjC4v8E+#8_Naso8G0oy<4KIIO=-s{l(7p|6D16lV&{>PBm%RP#S zxIlY=dzX8R`vB2>n|lKO#-aWMC?7=gKLL$Tz?UXO=M&JIxnDusUbIsOAoi0adbQwH}Zlpo_bZY%eH5LLp>Llks~#^Y$K zg|JzRwEH#q{z3YyM!bH;ZQ=eMQhyAZHxctg@FYk2dlT*KuiwMf!J}@}XF#_fmkLXr zToE@DQN3TX;HPMn#}MiJpzk-xj4DtM0rwT+yq?03r5>&Y@XuVa6wklG!i8KT zY<-Yh##Ks|-iOFeMjI~W3XrR(&}vzTTpIF&T51z_Cw!j?Yeh)82l+7#5voLs^04iD z$mcRdYCc-s!!>Xw?hLfLQ1>@P@i%Div+(i`ZZ0yaiBoaEMf6?<<#H|o`Lc~W#uakU za;@AHZZmRxJFNc%Sv&)o(S{!RKH~IeWXC6H@58X91akH0-v^M*c5V>X+zG4yH-?Gt zN)k50s!ho0v)onGUqSqTiJTOXg^waC9|1(@qbSdXXUn;Z+!tIXEKi5`OVC5tA&;h` zU&q6u^XQc&pj`|}(P*7gc={OZ`3dr2Dteg)@%vKx`wuCHmO^us5T8Z#H5jdqcO>psZBN}C+V`Z*;w=ujrt^m)?`Y?!eX{DS=0(iCS#_? lsL^Q?M4WnM>-0jh(Ih5(V*yxDQQbD5-bRgY>5&l;{}0jGOh^C# literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin new file mode 100644 index 0000000000000000000000000000000000000000..8b02fda7f5a761cc4c95fb90693353fb5a40bd96 GIT binary patch literal 73 zcmZQ#U|)JVn!gA0%GR0JVOS?3JwMiMn;erSOP>dSV1{PKpLpZh>)IVn!gA0b-W2#2iCDQIFJ|oP33#5QTumy(o`|1IzFg6BMvPf T3k3Lq7-RxF5Q9uGW@H2aY4{Ar literal 0 HcmV?d00001 diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin b/backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin new file mode 100644 index 0000000000000000000000000000000000000000..3b971412c9c43e57f3578fccde48706012b93118 GIT binary patch literal 153 zcmZQ%U|^UC#Ed}P3B+)~5?Y*E1mcM^aEWsBFfuW-u(Gjpa7yv=F^F?>Gjj9uatm;a g02MH List + /// Decode `blob`, re-encode it with today's writer, decode that, and compare. Raises if the + /// blob cannot be decoded at all, which is itself the failure. + reread : byte[] -> bool + } + + let private kindEq + (name : string) + (ser : 'T -> byte[]) + (deser : byte[] -> 'T) + (eq : 'T -> 'T -> bool) + (values : List<'T>) + : Kind = + { name = name + blobs = fun () -> values |> List.map ser + reread = fun blob -> let v = deser blob in eq v (deser (ser v)) } + + let private kind name ser deser values = kindEq name ser deser (=) values + + /// Everything that goes INTO a store, which is what a migration has to carry forward. The op log + /// first, because it is canonical and the rest is projection. + let kinds : List = + [ kind + "packageOp" + (BS.PT.PackageOp.serialize "corpus") + (BS.PT.PackageOp.deserialize "corpus") + Values.ProgramTypes.packageOps + kind + "ptPackageType" + (fun (t : PT.PackageType.PackageType) -> + BS.PT.PackageType.serialize t.hash t) + (BS.PT.PackageType.deserialize "corpus") + Values.ProgramTypes.packageTypes + kind + "ptPackageValue" + (fun (v : PT.PackageValue.PackageValue) -> + BS.PT.PackageValue.serialize v.hash v) + (BS.PT.PackageValue.deserialize "corpus") + Values.ProgramTypes.packageValues + kind + "ptPackageFn" + (fun (f : PT.PackageFn.PackageFn) -> BS.PT.PackageFn.serialize f.hash f) + (BS.PT.PackageFn.deserialize "corpus") + Values.ProgramTypes.packageFns + kind + "packageLocation" + (BS.PT.PackageLocation.serialize "corpus") + (BS.PT.PackageLocation.deserialize "corpus") + Values.ProgramTypes.packageLocations + kind + "toplevel" + (fun (tl : PT.DB.T) -> BS.PT.Toplevel.serialize tl.tlid tl) + (BS.PT.Toplevel.deserialize 0UL) + Values.ProgramTypes.toplevels + // The RT side is stored too: `package_values.rt_dval` and the compiled fn bodies. A migration + // that carried the op log forward and left these unreadable would look like it worked. + // + // NaN needs its own arm, as it does in the roundtrip tests above: `DFloat nan = DFloat nan` is + // false, so plain equality reports a perfectly good decode as a mis-decode. + kindEq + "rtDval" + (BS.RT.Dval.serialize "corpus") + (BS.RT.Dval.deserialize "corpus") + (fun a b -> + match a, b with + | RT.DFloat f1, RT.DFloat f2 when + System.Double.IsNaN f1 && System.Double.IsNaN f2 + -> + true + | _ -> a = b) + (Values.RuntimeTypes.dvals ()) + kind + "rtPackageValue" + (fun (v : RT.PackageValue.PackageValue) -> + BS.RT.PackageValue.serialize v.hash v) + (BS.RT.PackageValue.deserialize "corpus") + Values.RuntimeTypes.packageValues + kind + "rtPackageFn" + (fun (f : RT.PackageFn.PackageFn) -> BS.RT.PackageFn.serialize f.hash f) + (BS.RT.PackageFn.deserialize "corpus") + Values.RuntimeTypes.packageFns + kind + "rtInstructions" + (BS.RT.Instructions.serialize "corpus") + (BS.RT.Instructions.deserialize "corpus") + Values.RuntimeTypes.instructions ] + + /// One file per (version, kind), holding every blob for it: `[count][len][bytes]...`, all + /// little-endian uint32. A file per blob would be seven hundred files for the Dval set alone, + /// which makes the corpus unreadable as a diff and unpleasant to carry. + let private frame (blobs : List) : byte[] = + use stream = new System.IO.MemoryStream() + use w = new System.IO.BinaryWriter(stream) + w.Write(uint32 (List.length blobs)) + blobs + |> List.iter (fun b -> + w.Write(uint32 b.Length) + w.Write b) + w.Flush() + stream.ToArray() + + let private unframe (data : byte[]) : List = + use stream = new System.IO.MemoryStream(data) + use r = new System.IO.BinaryReader(stream) + let count = r.ReadUInt32() |> int + [ for _ in 1..count -> r.ReadBytes(r.ReadUInt32() |> int) ] + + let private fileFor (v : uint32) (k : Kind) = $"corpus/v{v}/{k.name}.bin" + + /// Write this build's blobs for the CURRENT version, for any kind not already stored. + /// + /// Never overwrites. A corpus entry is a historical fact about what some build actually wrote, and + /// regenerating one turns the net into a mirror -- which is precisely how the golden files above + /// stop covering an old format. Deliberately changing what a version's bytes are means deleting + /// the file by hand, and having to justify it. + let generate () : unit = + let v = BaseFormat.currentVersion + + System.IO.Directory.CreateDirectory(Config.serializationDir + $"corpus/v{v}") + |> ignore + + kinds + |> List.iter (fun k -> + let f = fileFor v k + if not (File.fileExists Config.Serialization f) then + File.writefileBytes Config.Serialization f (frame (k.blobs ())) + print $" corpus: wrote {f}") + + let tests = + // Every version from 1 up to this build's, since every one of them is a layout this binary + // claims to read. A version with no file fails rather than being skipped quietly. + [ 1u .. BaseFormat.currentVersion ] + |> List.map (fun v -> + testList + $"v{v}" + (kinds + |> List.map (fun k -> + test k.name { + let f = fileFor v k + + Expect.isTrue + (File.fileExists Config.Serialization f) + $"v{v} has stored {k.name} blobs ({f}). Write missing ones with \ + DARK_CONFIG_SERIALIZATION_GENERATE_TEST_DATA=y and COMMIT them: a format this \ + build claims to read with no stored bytes is a claim nothing checks." + + let stored = File.readfileBytes Config.Serialization f |> unframe + + stored + |> List.iteri (fun i blob -> + Expect.isTrue + (k.reread blob) + $"{k.name}[{i}] written at v{v} still decodes to the same value through this \ + build's writer") + + // At the CURRENT version there is a second, stronger thing to say: this build must + // still produce these exact bytes. That is what catches a layout change made without + // bumping `currentVersion`, which is the one format bug that corrupts silently -- two + // builds both calling themselves v1 and disagreeing about what a v1 blob means. + // + // A PREFIX, so a new test value can be appended without destroying the stored bytes + // for a version. Inserting one in the middle breaks this on purpose: the corpus is + // positional, and history is what it is for. + if v = BaseFormat.currentVersion then + let current = k.blobs () + + Expect.isGreaterThanOrEqual + (List.length current) + (List.length stored) + $"{k.name} still has at least the {List.length stored} test values its corpus was \ + written from. Removing one leaves stored bytes nothing can be compared against; \ + append rather than insert or delete." + + List.zip stored (List.truncate (List.length stored) current) + |> List.iteri (fun i (storedBlob, currentBlob) -> + Expect.equal + currentBlob + storedBlob + $"this build writes {k.name}[{i}] exactly as v{v} did. If the layout genuinely \ + changed, bump BaseFormat.currentVersion and keep a readVN -- do not rewrite \ + this file.") + }))) + + let generateTestFiles () = // Enabled in dev so we can see changes as git diffs // Disabled in CI so changes will fail the tests if Config.serializationGenerateTestData then ConsistentSerializationTests.generateTestFiles () + Corpus.generate () () @@ -351,4 +562,6 @@ let tests = RT.closureAccessIsStripped RT.namedFnAccessIsStripped ] - testList "consistent serialization" ConsistentSerializationTests.testTestFiles ] + testList "consistent serialization" ConsistentSerializationTests.testTestFiles + + testList "stored blobs from past formats" Corpus.tests ] diff --git a/packages/darklang/cli/registry.dark b/packages/darklang/cli/registry.dark index da06007ce0..4d603a3253 100644 --- a/packages/darklang/cli/registry.dark +++ b/packages/darklang/cli/registry.dark @@ -82,6 +82,7 @@ module Registry = ("db", "Manage databases", [], Packages.DB.execute, Packages.DB.help, Packages.DB.complete) ("outliner", "Interactive tree outliner", [ "tree-exp" ], Outliner.App.execute, Outliner.App.help, Outliner.App.complete) ("export-seed", "Export minimal seed.db from current database", [], ExportSeed.execute, ExportSeed.help, ExportSeed.complete) + ("store", "What this store is, and moving it between op-log formats", [], Store.execute, Store.help, Store.complete) ("traces", "List and view execution traces", [ "trace" ], Tracing.execute, Tracing.help, Tracing.complete) ("review", "Stage someone else's ops for op-level approval", [], Cli.Sync.executeReviewCmd, Cli.Sync.helpReview, Cli.Sync.complete) ("conflicts", "Review what a merge, rebase or sync auto-resolved", [], Conflicts.execute, Conflicts.help, Conflicts.complete) @@ -211,7 +212,7 @@ module Registry = ("Security", [ "permissions" ]) ("Network", [ "devices" ]) ("Apps", [ "apps" ]) - ("Utilities", [ "clear", "help", "docs", "quit", "builtins", "export-seed", "views", "workbench" ]) ] + ("Utilities", [ "clear", "help", "docs", "quit", "builtins", "export-seed", "store", "views", "workbench" ]) ] let formatCommandGroup (groupName: String) (commandNames: List) (commands: List) : String = let formattedCommands = diff --git a/packages/darklang/cli/store.dark b/packages/darklang/cli/store.dark new file mode 100644 index 0000000000..ce665e6bfb --- /dev/null +++ b/packages/darklang/cli/store.dark @@ -0,0 +1,145 @@ +module Darklang.Cli.Store + +// `dark store` -- what this store IS, and moving it between op-log formats. +// +// Distinct from `dark backups`, which is about copies of a store, and from `dark status`, which is +// about the code in one. This is about the FILE: which format its ops are written in, which commit +// it was cut at, which build cut it. +// +// After the flip there is no `packages/` to rebuild a store from, so a format change has to carry +// the store forward in place. `upgrade` is that, and it is a no-op until the first real bump -- it +// exists now so that the bump is not also the first time the migration runs. + +let help (_state: AppState) : String = + [ "What this store is, and moving it between op-log formats." + "" + "Usage:" + " store what this store says about itself" + " store upgrade move the op log to the format this build writes" + " store rollback put back the copy `upgrade` took on its way to format " + "" + "`upgrade` copies the store first and rewrites in one transaction. It only rewrites" + "blobs: a change that moves content hashes is a larger migration, and it refuses" + "rather than half-doing it." + "" + "Both write, so both need --yes when nothing is watching." ] + |> Stdlib.String.join "\n" + + +/// `dark store` with no arguments: the stamp, as it stands. +let showInfo (state: AppState) : AppState = + let rows = SCM.StoreMeta.all () + + if rows == [] then + Stdlib.printLine + "This store carries no stamp. It predates one; `store upgrade` writes it." + + state + else + rows + |> Stdlib.List.iter (fun row -> + let (k, v) = row + let key = Stdlib.String.padEndToWidth k 8 + Stdlib.printLine $"{key} {v}") + + state + + +let executeUpgrade (state: AppState) (args: List) : AppState = + let (autoConfirm, _rest) = Packages.Remove.parseArgs args + + // A whole-log rewrite, so it asks. `confirmExplicit` rather than `confirm`: a stray Return must + // not start one, and an unattended run says `-y`. + if + Stdlib.Bool.not ( + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + "Rewrite every op in this store into this build's format? A copy is taken first. (y/n): ") + ) + then + state + else + match Stdlib.LocalStore.upgrade () with + | Error e -> + Stdlib.printLine (View.formatError e) + state + | Ok report -> + Stdlib.printLine + $"Upgraded from format {Stdlib.Int64.toString report.from} to {Stdlib.Int64.toString report.to}." + + Stdlib.printLine + $"{Stdlib.Int64.toString report.rewritten} op(s) rewritten; the previous store is at {report.backup}." + + if report.unreadable > 0L then + Stdlib.printLine + $"{Stdlib.Int64.toString report.unreadable} op(s) this build cannot read were left as they are, so a later build can still apply them." + else + () + + Stdlib.printLine "Projections were re-folded. Nothing further to do." + state + + +let executeRollback (state: AppState) (args: List) : AppState = + match args with + | [] -> + Stdlib.printLine (View.formatError "Usage: store rollback ") + state + | target :: rest -> + match Stdlib.Int64.parse target with + | Error _ -> + Stdlib.printLine + (View.formatError + $"`store rollback` takes the format version that was upgraded TO, and \"{target}\" isn't a number") + + state + | Ok n -> + let (autoConfirm, _rest) = Packages.Remove.parseArgs rest + + if + Stdlib.Bool.not ( + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + $"Replace this store with the copy taken before the move to format {target}? (y/n): ") + ) + then + state + else + match Stdlib.LocalStore.rollbackTo n with + | Error e -> + Stdlib.printLine (View.formatError e) + state + | Ok path -> + Stdlib.printLine $"Restored from {path}. Restart to pick it up." + state + + +let execute (state: AppState) (args: List) : AppState = + match args with + | [] -> showInfo state + | [ "help" ] -> + Stdlib.printLine (help state) + state + | "upgrade" :: rest -> executeUpgrade state rest + | "rollback" :: rest -> executeRollback state rest + | other -> + Stdlib.printLine + (View.formatError + $"`store` takes nothing, `upgrade` or `rollback`; not \"{Stdlib.String.join other " "}\"") + + Stdlib.printLine (help state) + state + + +let complete (_state: AppState) (args: List) : List = + match args with + | [] -> [ "upgrade", "rollback", "help" ] |> Stdlib.List.map Completion.simple + | [ partial ] -> + [ "upgrade", "rollback", "help" ] + |> Stdlib.List.filter (fun c -> Stdlib.String.startsWith c partial) + |> Stdlib.List.map Completion.simple + // `rollback` takes a format version, which only the store knows and only one of which is ever + // right, so there is nothing useful to offer past the subcommand. + | _ -> [] diff --git a/packages/darklang/stdlib/localStore.dark b/packages/darklang/stdlib/localStore.dark index f3e278c03f..e3dd811ad0 100644 --- a/packages/darklang/stdlib/localStore.dark +++ b/packages/darklang/stdlib/localStore.dark @@ -62,3 +62,52 @@ let seedTo (upToCommit: Stdlib.Option.Option) : Stdlib.Result.Result = Builtin.pmSeedExport path upToCommit + + +/// What an upgrade did. Structured rather than a sentence: a caller decides what to print, and +/// `unreadable > 0` is a thing a script might want to act on. +type UpgradeReport = + { /// The format the store was in. + from: Int64 + /// The format it is in now, which is the one this build writes. + to: Int64 + /// Ops decoded and written back out. + rewritten: Int64 + /// Ops this build cannot read at all: a peer's newer format, stored inert. Left exactly as + /// they are, because re-encoding is impossible and dropping them would lose work a later + /// build can still apply. + unreadable: Int64 + /// Where the pre-upgrade store was copied to, which is what puts back. + backup: String } + + +/// Move this store's op log to the format this build writes. +/// +/// A no-op until the first format bump: with nothing to migrate between it says so rather than +/// touching anything. The copy lands before any write, and the rewrite is one transaction, because +/// a half-converted store has some ops in one format and some in another with nothing recording +/// which -- and no reader can sort that out afterwards. +/// +/// Only rewrites BLOBS. A change that moves content hashes is a different and much larger +/// migration; this refuses rather than half-doing it. See `LibDB.StoreUpgrade`. +let upgrade () : Stdlib.Result.Result = + match Builtin.pmStoreUpgrade () with + | Error e -> Stdlib.Result.Result.Error e + | Ok parts -> + let (from, to, rewritten, unreadable, backup) = parts + + (UpgradeReport + { from = from + to = to + rewritten = rewritten + unreadable = unreadable + backup = backup }) + |> Stdlib.Result.Result.Ok + + +/// Put back the copy took on its way to . +/// +/// Contents, not the file, so connections already open keep working. Anything already read into +/// memory is still the upgraded store, so the caller should say to restart. +let rollbackTo (target: Int64) : Stdlib.Result.Result = + Builtin.pmStoreRollback target diff --git a/scripts/testing/_gates-store b/scripts/testing/_gates-store index 803424ec42..35d6732e36 100644 --- a/scripts/testing/_gates-store +++ b/scripts/testing/_gates-store @@ -264,3 +264,100 @@ gate_reload_is_reproducible() { fi echo "reload-is-reproducible: two reloads of one tree agree on all $n op ids." } + +# --- store-upgrade: a format bump, carried in place --------------------------------------------- +# There is only ONE op-log format so far, so the migrator has nothing real to migrate between. +# `DARK_FORMAT_VERSION` is the synthetic bump: the binary writes the version named and reads every +# version up to it, so the layout is identical and a migration across it is exactly the +# format-only case -- a blob rewrite with no identity moving. Which is the case worth having +# mechanised and tested BEFORE the first real bump, rather than discovering during one. +# +# Gate, not an F#/.dark test: the format version is read at process start, so "the same store seen +# by a build that writes a different version" needs a different PROCESS, twice. +gate_store_upgrade() { + set -uo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + ROOT=rundir/store-upgrade-test + pass=0; fail=0 + ok() { pass=$((pass+1)); printf ' ok %s\n' "$1"; } + bad() { fail=$((fail+1)); printf ' FAIL %s\n got: %s\n' "$1" "$(head -c 300 <<<"$2" | tr '\n' ' ')"; } + want() { if grep -qE "$2" <<<"$3"; then ok "$1"; else bad "$1" "$3"; fi; } + + rm -rf "$ROOT"; mkdir -p "$ROOT/home" + scripts/testing/_copy-store "$ROOT/data.db" + + # v1: this build's own version. v2: the same build pretending to write one format on. + v1() { DARK_CONFIG_RUNDIR="$PWD/$ROOT" HOME="$PWD/$ROOT/home" "$PWD/$CLI" "$@" 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; } + v2() { DARK_FORMAT_VERSION=2 DARK_CONFIG_RUNDIR="$PWD/$ROOT" HOME="$PWD/$ROOT/home" "$PWD/$CLI" "$@" 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; } + ops() { sqlite3 "$ROOT/data.db" 'SELECT count(*) FROM package_ops;' 2>/dev/null || echo -1; } + fmt() { sqlite3 "$ROOT/data.db" "SELECT value FROM store_meta WHERE key = 'format';" 2>/dev/null || echo ""; } + + echo "== before" + v1 store > /dev/null + want "the store stamps itself format 1 at open" "^1$" "$(fmt)" + want "and it works" "^4$" "$(v1 eval '(Stdlib.List.length [1L, 2L, 3L, 4L]) |> Stdlib.Int.toString' | tail -1)" + + # A name authored BEFORE the upgrade, to check the rewrite carried meaning and not just bytes. + v1 fn /SU.T.before '() : Int64 = 1717L' > /dev/null + v1 commit "store-upgrade gate" -y > /dev/null + want "a fn authored at v1 resolves" "^1717$" "$(v1 eval 'SU.T.before ()' | tail -1)" + after_author_ops=$(ops) + + echo "== a build that writes the same format has nothing to do" + want "and says so rather than rewriting" "already format 1" "$(v1 store upgrade -y)" + + echo "== an id that does not re-derive stops it dead" + # The line between the two migrations, forged. A store whose ids were minted by a different + # HASHING needs the whole log re-minted, not its blobs rewritten, and the only way to tell is to + # recompute each op's id and compare. So: an op filed under an id its own content does not + # produce. The upgrade must refuse and touch nothing, because half-doing this is unrecoverable. + sqlite3 "$ROOT/data.db" "INSERT INTO package_ops (id, op_blob, applied, effective, commit_hash, origin_ts) + SELECT '00000000-0000-4000-8000-00000000ffff', op_blob, applied, effective, commit_hash, origin_ts + FROM package_ops LIMIT 1;" + forged_ops=$(ops) + out=$(v2 store upgrade -y) + want "it refuses" "re-derives a different id" "$out" + want "and names it an identity-changing migration" "identity-changing migration" "$out" + want "and says the store is untouched" "store is untouched" "$out" + want "the stamp did not move" "^1$" "$(fmt)" + want "and nothing was rewritten" "^$forged_ops$" "$(ops)" + sqlite3 "$ROOT/data.db" "DELETE FROM package_ops WHERE id = '00000000-0000-4000-8000-00000000ffff';" + + echo "== the upgrade" + out=$(v2 store upgrade -y) + want "it reports the move" "Upgraded from format 1 to 2" "$out" + want "it rewrote the whole log" "$after_author_ops op\(s\) rewritten" "$out" + want "it says where the old store is" "pre-v2" "$out" + want "the stamp moved" "^2$" "$(fmt)" + want "no op was lost" "^$after_author_ops$" "$(ops)" + if [[ -f "$ROOT/data.db.pre-v2" ]]; then ok "the pre-upgrade copy is on disk"; else bad "the pre-upgrade copy is on disk" "$(ls "$ROOT")"; fi + + echo "== and the store still means what it meant" + want "the fn authored at v1 still resolves" "^1717$" "$(v2 eval 'SU.T.before ()' | tail -1)" + want "so does the package set" "^4$" "$(v2 eval '(Stdlib.List.length [1L, 2L, 3L, 4L]) |> Stdlib.Int.toString' | tail -1)" + want "authoring still works after it" "^2727$" "$(v2 fn /SU.T.after '() : Int64 = 2727L' > /dev/null; v2 eval 'SU.T.after ()' | tail -1)" + want "a second upgrade is a no-op" "already format 2" "$(v2 store upgrade -y)" + + echo "== a build that reads an older format than the store" + # The one thing that must not happen quietly. It opens (so a rollback is reachable) and says + # which way out there is. + # It dies on the first package lookup -- resolving the name of the command you typed is one -- + # so what it must do is SAY so, and name a file a person can move without a working binary. + want "it says the store is ahead of it" "format 2 and this build reads 1" "$(v1 store)" + want "and names the file to move back" "pre-v2 -- move it back over" "$(v1 store)" + want "and does not downgrade the stamp behind your back" "^2$" "$(fmt)" + + echo "== rollback" + want "it asks first" "" "$(v2 store rollback 2 < /dev/null)" + want "and rolling back needs the version" "takes the format version" "$(v2 store rollback banana -y)" + want "rollback restores" "Restored from" "$(v2 store rollback 2 -y)" + want "the stamp is back" "^1$" "$(fmt)" + want "this build reads it again" "^1717$" "$(v1 eval 'SU.T.before ()' | tail -1)" + want "and the op authored after the upgrade is gone with it" "not found|couldn't be found" "$(v1 eval 'SU.T.after ()' | tail -1)" + want "rolling back twice is refused, not repeated" "no pre-v9 copy" "$(v2 store rollback 9 -y)" + + echo + echo "passed: $pass failed: $fail" + [[ "$fail" -eq 0 ]] +} diff --git a/scripts/testing/_gates-sync b/scripts/testing/_gates-sync index 29a61ec63f..f2365dfa16 100644 --- a/scripts/testing/_gates-sync +++ b/scripts/testing/_gates-sync @@ -926,7 +926,7 @@ gate_seed_serving() { env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ "$PWD/$CLI" permissions allow package-read > /dev/null 2>&1 || true got=$(env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ - "$PWD/$CLI" eval '(Stdlib.List.length [1L, 2L, 3L]) |> Stdlib.Int64.toString' 2>&1 | tail -1 || true) + "$PWD/$CLI" eval '(Stdlib.List.length [1L, 2L, 3L]) |> Stdlib.Int.toString' 2>&1 | tail -1 || true) if [[ "$got" == *3* ]]; then echo " ok a store grown from the fetched seed resolves names and evaluates" else diff --git a/scripts/testing/gates b/scripts/testing/gates index 7fb8096663..b9cab35878 100755 --- a/scripts/testing/gates +++ b/scripts/testing/gates @@ -32,14 +32,18 @@ GATES=( workbench-views first-day backups-restore + store-upgrade reload-is-reproducible gates-are-clean ) -# What CI runs, in one config line: the three gates it has always run, the ones that need only -# the debug build already in backend/Build. first-day wants a published artifact and the sync -# gates add minutes; those run locally, by name or via `all`. -CI_GATES=(relay-routes server-folds seed-serving workbench-scm workbench-views) +# What CI runs, in one config line: the ones that need only the debug build already in +# backend/Build. first-day wants a published artifact and the sync gates add minutes; those run +# locally, by name or via `all`. +# +# `store-upgrade` is here rather than local-only because it is the migrator's only test, and a +# migration that has quietly stopped working is discovered at the worst possible moment. +CI_GATES=(relay-routes server-folds seed-serving store-upgrade workbench-scm workbench-views) describe() { # shellcheck disable=SC2016 # the backticks are prose, not expansion @@ -55,6 +59,7 @@ describe() { workbench-views) echo 'every workbench view, one session each' ;; first-day) echo 'a published binary, empty home, whole SCM workflow' ;; backups-restore) echo 'backup, wipe, restore; the destructive verbs ask first' ;; + store-upgrade) echo 'a synthetic format bump, carried in place, and rolled back' ;; reload-is-reproducible) echo 'two reloads of one tree agree on every op id' ;; gates-are-clean) echo 'no gate touches the shared dev store (slow: re-runs the rest)' ;; *) echo '?' ;; From 210a806a12c47ab4a08f7ccf4dfd3c61ea0c6421 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 12:32:38 -0400 Subject: [PATCH 14/32] Correct the docs the server change made wrong, and say where a package set comes from `dark docs relay` said the server never folds what it hosts. It does now, into main, like any other instance; what it does not do is RUN it. The old text had that exactly backwards, which is worse than silence on a page people read to decide whether pointing a machine at a server is safe. It now also carries the `/seed/*` routes and the honest sentence: a push here can rebind what this server serves, and the write secret is currently the whole of the answer. `dark docs packages` gains the two answers to "where does this build's package set come from", the pin commands, and what a store says about itself. `AGENTS.md`: the `package-ref-hashes.txt` gotcha now says it is a projection of the store and names `refs generate`, which is how a seed-mode store produces it with no `packages/` to reload. And a section next to the build loop on the package set, which says plainly that the pinned path is written and unverified. --- AGENTS.md | 13 +++++++ packages/darklang/cli/docs/command.dark | 2 +- packages/darklang/cli/docs/packages.dark | 35 +++++++++++++++++- packages/darklang/cli/docs/relay.dark | 46 ++++++++++++++++++++---- 4 files changed, 88 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0f20f8aa85..a014d00d87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,19 @@ alter the serialized package format, and there's no cheap way to ask whether thi did. Narrowing it is the biggest remaining win in the loop, and it's entangled with `package-ref-hashes.txt`, so coordinate before starting. +## Where the package set comes from + +`package-set.txt` at the root says which of two, and it ships `commit unset`, which +means the first: + + commit unset built from `packages/` by reloading it, as always + commit fetched as a seed from a package server, at that commit + +`scripts/build/prepare-package-set` is the one place that answers that question, and +CI's package-reloading jobs go through it. `scripts/packages/pin` writes the pin; +`dark docs packages` has the rest. The pinned path has never run against a deployed +server, so treat it as written-and-unverified until it has. + The container builds once when it starts. Rebuild-on-save is available but off by default, because a five-file change under a watcher pays for five rebuilds, four of them on half-finished states that produce real-looking failures: diff --git a/packages/darklang/cli/docs/command.dark b/packages/darklang/cli/docs/command.dark index 2d0abcf072..9843dc0696 100644 --- a/packages/darklang/cli/docs/command.dark +++ b/packages/darklang/cli/docs/command.dark @@ -150,7 +150,7 @@ module Topics = DocTopic { name = "relay" - description = "The optional sync server: setup, the write secret, smoke tests" + description = "The optional sync server: setup, the write secret, serving a seed" aliases = [ "matter", "server", "sync-server" ] context = "cli" content = Docs.Relay.content diff --git a/packages/darklang/cli/docs/packages.dark b/packages/darklang/cli/docs/packages.dark index 8f75a93265..2eb41b5de8 100644 --- a/packages/darklang/cli/docs/packages.dark +++ b/packages/darklang/cli/docs/packages.dark @@ -34,4 +34,37 @@ or search queries. Write `Stdlib.List.map` (preferred) or the full source name ## Stdlib vs Builtin Stdlib: packages/darklang/stdlib/*.dark Builtins: F# in backend/src/Builtins/ - Use `builtins` to list all builtins""" + Use `builtins` to list all builtins + +## Where a build's package set comes from + +Two answers, and `package-set.txt` at the repo root says which: + + commit unset built from `packages/` on disk, by reloading it + commit fetched as a SEED from a package server, at that commit + +The second is the direction this is going: the server's store is the source of +truth, a seed of it is what a fresh container and CI fetch, and `packages/` goes +away. Today it ships `unset`, so nothing has changed yet. + + scripts/packages/pin head pin to what that server has now + scripts/packages/pin pin to a named commit + scripts/packages/pin --show what is pinned + scripts/packages/pin --unset back to building from `packages/` + +The server address is not in the repo: it comes from `DARK_SEED_URL` or +`--url`. A pin says WHICH package set, not where to get it. + + scripts/build/prepare-package-set get it, whichever of the two it is + scripts/fetch-seed --url just the fetch + +## What a store says about itself + + store format, which commit it was cut at, which build cut it + store upgrade move the op log to the format this build writes + store rollback put back the copy `upgrade` took on its way to format + +`upgrade` copies the store first and rewrites in one transaction. It only +rewrites blobs: a change that moves content hashes is a much larger migration, +and it refuses rather than half-doing it. It is a no-op until the first format +bump.""" diff --git a/packages/darklang/cli/docs/relay.dark b/packages/darklang/cli/docs/relay.dark index ad23f4c38a..1dce35db05 100644 --- a/packages/darklang/cli/docs/relay.dark +++ b/packages/darklang/cli/docs/relay.dark @@ -4,11 +4,22 @@ let content () : String = """# Running your own sync server (the relay) Syncing is OPTIONAL and opt-in: an instance is complete without one, and -connecting is one `dark sync setup`. The relay is the sync server: a dumb -store-and-forward for the op log that holds ops and hands them back out. It never folds or runs what it hosts, so pushing code to it -cannot change what it serves, and the worst a bad relay can do is withhold. -Losing one loses availability, not history -- every client holds the full log -it has seen, and the next push repopulates a wiped relay. +connecting is one `dark sync setup`. The server holds the op log and hands it +back out: `/sync/pull` pages it, `/seed/*.db` cuts it as a file. Losing one +loses availability, not history -- every client holds the full log it has seen, +and the next push repopulates a wiped server. + +It FOLDS what you push, into main, like any other instance. That is what makes +`/m`, `/p` and the seed routes show hosted packages rather than only what the +binary shipped with. It does not RUN them: a pushed `val` is folded, browsable +and servable in a seed, and never evaluated on the server -- whoever fetches it +evaluates it on their own machine, under their own policy. + +Folding still binds NAMES, though, last-writer-wins over the whole store, and +the server resolves its own router through those names. So anyone who can push +here can change what this server serves. Who may bind what is not answered yet; +for now the write secret is the whole of the answer, so treat push access as +trust. ## Standing one up @@ -27,11 +38,33 @@ Clients connect with: curl https:///ping answers curl https:///sync/head op count, as JSON + curl https:///seed/meta head commit, format, counts curl -X POST https:///sync/push 401 without the secret The 401 from another machine is the smoke test worth doing once: it proves the secret is required from the outside, not just missing locally. +## Serving the package set as a file + + /seed/latest.db the store as of now + /seed/.db the store as of that commit and its ancestors + /seed/meta the same four fields without the megabytes + /seed/meta?commit= ...about that commit, or 404 if it is unknown + +A seed is the op log as a file: what a fresh install folds to become this +store, and what CI fetches instead of parsing text. `latest.db` resolves to +the head and is served as the cut AT that head, so it is the same thing as +asking for that commit by name. + +Cut at a commit, always, because that is what makes a pin reproducible: the +same commit yields the same ops however far the store has moved since. Cuts +are cached beside the store and never need invalidating, commits being +immutable and ops append-only. A short commit prefix resolves like git's; an +ambiguous one is refused rather than guessed at. + +Clients fetch with `scripts/fetch-seed --url `, which takes the commit +from `package-set.txt` if that tree has a pin. See `dark docs packages`. + ## Two things not to do to a relay - Don't run client commands (`dark status`, `dark fn`, ...) against its @@ -48,7 +81,8 @@ the secret is required from the outside, not just missing locally. are idempotent, run `dark sync` a conflict after pull not data loss -- both versions are kept; `dark conflicts` - "store was written by a newer build" upgrade dark; the store is fine + "this store is format N" upgrade dark, or move the + `.pre-vN` copy back; `dark store` Reads are public by design. If that's wrong for your relay, put it behind something that terminates auth; the relay itself only gates writes and From 94fac03768432ffce39bcc24cce1f509000fcf29 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 12:46:53 -0400 Subject: [PATCH 15/32] Make lsp-branches grant what it needs instead of inheriting it The policy is per INSTANCE, and a gate is its own instance, so this one was depending on whatever the dev instance happened to allow. It passed or failed by machine. The symptom is worth knowing, because it names nothing: the LSP writes a debug log, a denied effect raises, and a raise inside the server's request loop takes the loop down. So every request goes unanswered and the gate reports the first assertion after that, which was about branch names. `LspServer.logFilePath` is a hardcoded absolute container path. That is its own bug -- it ignores the rundir, and on an installed machine the directory cannot exist -- and not this branch's to fix; granting it by name is the honest way to depend on it until it moves. --- scripts/testing/_gates-ui | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/testing/_gates-ui b/scripts/testing/_gates-ui index a36cf7806f..ea9230b8ac 100644 --- a/scripts/testing/_gates-ui +++ b/scripts/testing/_gates-ui @@ -136,6 +136,18 @@ gate_lsp_branches() { sqlite3 "$WORK/data.db" 'DELETE FROM config_v0;' 2>/dev/null || true export DARK_CONFIG_RUNDIR="$PWD/$WORK" + + # The policy is per INSTANCE, and this gate is its own instance, so it has to grant what the + # server needs rather than inherit whatever the dev instance happens to allow. It used to + # inherit, which is why it passed on some machines and not others: the LSP writes a debug log, + # a denied effect raises, and a raise inside the server's loop takes the loop down -- so the + # gate's only symptom was every request going unanswered. + # + # `LspServer.logFilePath` is a hardcoded absolute container path, which is its own bug and not + # this gate's to fix; granting it by name is the honest way to depend on it until it moves. + "$PWD/$CLI" permissions allow file write "/home/dark/app/rundir/logs/lsp-server.log" >/dev/null 2>&1 + "$PWD/$CLI" permissions allow package-write >/dev/null 2>&1 + "$PWD/$CLI" branch create lsp-gate >/dev/null 2>&1 "$PWD/$CLI" --branch lsp-gate fn LspGate.T.f '() : Int64 = 5L' >/dev/null 2>&1 From fa29c35fcc4e106b1d429e180fbc0d171e4e3118 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 12:49:46 -0400 Subject: [PATCH 16/32] Say where the way back is, in the place someone looks for it `dark backups` does not list the copy an upgrade takes, so after one went wrong there was nothing that named it. `dark store` now points at `store rollback ` whenever the format is above 1. The verb knows its own filename and says plainly when there is no copy to put back, so this is a pointer rather than a second source of truth about what exists. --- packages/darklang/cli/store.dark | 13 +++++++++++++ scripts/testing/_gates-store | 1 + 2 files changed, 14 insertions(+) diff --git a/packages/darklang/cli/store.dark b/packages/darklang/cli/store.dark index ce665e6bfb..483db1cc37 100644 --- a/packages/darklang/cli/store.dark +++ b/packages/darklang/cli/store.dark @@ -42,6 +42,19 @@ let showInfo (state: AppState) : AppState = let key = Stdlib.String.padEndToWidth k 8 Stdlib.printLine $"{key} {v}") + // Naming the verb here, because this is where someone looks after an upgrade went wrong and + // `dark backups` does not list the copy an upgrade takes. The verb knows its own filename; it + // says so plainly if there is no copy to put back. + let format = SCM.StoreMeta.get "format" + + if (format != "") && (format != "1") then + Stdlib.printLine "" + + Stdlib.printLine + $" `store rollback {format}` puts back the copy taken on the way here." + else + () + state diff --git a/scripts/testing/_gates-store b/scripts/testing/_gates-store index 35d6732e36..1a39512e4b 100644 --- a/scripts/testing/_gates-store +++ b/scripts/testing/_gates-store @@ -330,6 +330,7 @@ gate_store_upgrade() { want "it rewrote the whole log" "$after_author_ops op\(s\) rewritten" "$out" want "it says where the old store is" "pre-v2" "$out" want "the stamp moved" "^2$" "$(fmt)" + want "and \`store\` names the way back" "store rollback 2" "$(v2 store)" want "no op was lost" "^$after_author_ops$" "$(ops)" if [[ -f "$ROOT/data.db.pre-v2" ]]; then ok "the pre-upgrade copy is on disk"; else bad "the pre-upgrade copy is on disk" "$(ls "$ROOT")"; fi From 16648bbdf9dece0dd69234ed4aee52966bc85cd5 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 15:04:00 -0400 Subject: [PATCH 17/32] Track package-ref-hashes.txt Stachu's call, against my recommendation, and his reason is the better one: a kernel entry point changing identity should be visible in review, and `assert-clean-worktree` is what makes carrying the regenerated file non-optional. There is a second argument I had missed, and it is what turns this from a tradeoff into a check. On the PINNED path, `prepare-package-set` regenerates this file from the FETCHED store. If that differs from the committed one, the pin's package set is not the one this kernel was built against -- and with the file tracked, the build jobs' `assert-clean-worktree` says so. Untracked, that mismatch is silent, and it is exactly the failure a pin exists to prevent. The cost is real and stands: two branches that both touch packages conflict here, and GitHub's rebase cannot regenerate it. `AGENTS.md` now says how to resolve one -- regenerate, never hand-merge, since the lines are content hashes and picking sides is meaningless. Checked that a reload reproduces the committed file byte for byte, so the worktree stays clean unless one of the 206 actually moves. --- .circleci/config.yml | 18 +- .gitignore | 4 - AGENTS.md | 14 +- .../src/LibExecution/package-ref-hashes.txt | 206 ++++++++++++++++++ scripts/build/check-seed-carries-refs | 2 + 5 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 backend/src/LibExecution/package-ref-hashes.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index d6cde90d95..dec06af98a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -410,13 +410,17 @@ jobs: - rundir/seed.db # Hand over the hash file too, not just the seed. # - # `reload-packages` above populates it, but it is gitignored, so the - # build jobs' own checkout has only the empty one MSBuild creates. - # They then embed THAT, and every ref in the shipped binary resolves - # to "" -> `FnNotFound` on every command. It goes unnoticed inside a - # source tree, because there the file on disk wins over the embedded - # copy and the first run regenerates it. A user running the artifact - # anywhere else has only the embedded copy. + # It is tracked now, so a checkout already has it; this carries the + # one the step above just PRODUCED, which is the one that matches the + # seed beside it. On the pinned path those can differ, and when they + # do the difference is the finding: the pin's package set is not the + # one this kernel was built against, and `assert-clean-worktree` in + # the build jobs is what says so. + # + # Embedding a mismatched one is not a loud failure. Every ref in the + # shipped binary resolves to "" -> `FnNotFound` on every command, and + # only OUTSIDE a source tree, because inside one the file on disk + # wins over the embedded copy and the first run regenerates it. - backend/src/LibExecution/package-ref-hashes.txt # All four linux artifacts from one x64 runner. The cross-compiles work diff --git a/.gitignore b/.gitignore index 017e54b430..6f6a90ee56 100644 --- a/.gitignore +++ b/.gitignore @@ -24,10 +24,6 @@ backend/src/Wasm/publish/ backend/src/Wasm/wwwroot/packages.snapshot darklang-repl.zip -# Generated by reload-packages. Untracked on purpose: two branches that both touch packages -# produce conflicting hashes, and a rebase on GitHub cannot regenerate it -- you would have to pull, -# reload and commit by hand every time. -backend/src/LibExecution/package-ref-hashes.txt # Native SQLite archives for AOT — built by scripts/build/build-sqlite.sh # from the amalgamation, cached locally + in CI. diff --git a/AGENTS.md b/AGENTS.md index a014d00d87..8bffa87f94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -339,10 +339,16 @@ Empty is tolerated; non-empty with a missing key crashes at startup with "Packag hash not found". After adding a ref: `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages` -It is a PROJECTION of the store, not a source file, which is why it is not tracked. A store -that came from a seed has no `packages/` to reload, so regenerate it from the store instead: -`scripts/run-local-exec refs generate`. `scripts/build/prepare-package-set` does that on the -pinned path; see `package-set.txt`. +It IS tracked, and it is a projection of the store, which is the awkward combination it has to +be: committing it is what makes a kernel entry point changing identity visible in review, and +`assert-clean-worktree` is what enforces it. So a PR that moves one of the 206 hashes has to +carry the regenerated file. + +**Resolving a conflict in it: regenerate, never hand-merge.** Two branches that both touch +packages will conflict here, and the lines are content hashes, so picking sides is meaningless. +`git checkout --theirs` it, then `./scripts/build/reload-packages` (or, on a store that came +from a seed and has no `packages/` to reload, `scripts/run-local-exec refs generate`) and commit +what that produces. **Name resolution in test files.** `backend/testfiles/` is parsed with owner "Tests", so `Darklang.*` names need full qualification or the `Stdlib.` shortcut. `Stdlib.Json.ParseError.toString` diff --git a/backend/src/LibExecution/package-ref-hashes.txt b/backend/src/LibExecution/package-ref-hashes.txt new file mode 100644 index 0000000000..7ce58708a5 --- /dev/null +++ b/backend/src/LibExecution/package-ref-hashes.txt @@ -0,0 +1,206 @@ +fn/Cli.Terminal.renderValue|20a0dff54d9a7e4526e17aa4efe30b98c22df9a897858ddc6e5507bfcfef0724 +fn/Cli.executeCliCommand|a14ca72c5b47dc5bab5adc8777d36b4cb345e14383831e121f34343a10902b46 +fn/Internal.Test.parseSingleTestFromFile|408997505554a133d6f611ba95fcf82fe9f92c6c1b2164ddf1211c6a10caca75 +fn/LanguageTools.Parser.CliScript.parseForCli|941945f41a5765708ca908fb58e6129ab5843fbf3159e3b10cce2f703bcdf7a0 +fn/LanguageTools.Parser.Parse.parsePTExpr|70bea7c1a45c75f6185e1d3db5677e18b66adf91641cb442ea484e01d3b50ee4 +fn/LanguageTools.Parser.Parse.parsePTExprInContext|5c46e13fb98ec3d1d4b9956611de2b2884e97a9a964dce451c52f11f8e1cfa0c +fn/LanguageTools.Parser.Parse.parsePTSourceFileWithOps|578620c01e3e5f277fb36bf66219acba1b2e05ac3e31afc03a565e814c36641f +fn/PrettyPrinter.ProgramTypes.sourceFile|007d064ece21498a303caab46c3c40752dd30b4d11b76958b27ad5380579bfdc +fn/PrettyPrinter.ProgramTypes.sourceFileAtWidth|daadd234444f41787473b3edd814048e6648cbf307828213cbece6c7e89dd72d +fn/PrettyPrinter.RuntimeTypes.Dval.valueTypeName|d239f580e296aee0a1e2c34a6790386734168f298d0d8092d032a6baccb1e7e1 +fn/PrettyPrinter.RuntimeTypes.RuntimeError.toErrorMessage|7fd47bcec6e8f46519bd2794157aabceea5512da9fce19297b6e82d142d477fa +fn/PrettyPrinter.RuntimeTypes.RuntimeError.toString|cdc00e456e722be5dc507e18733ab004593af7ab7bdd5e8f943b47924824cb2c +fn/PrettyPrinter.RuntimeTypes.dval|887ac1c5b2c2bd2c0c33fe31cfd01bf8ba2084d39e2836bd2998dd0eb4a7ae0c +fn/PrettyPrinter.RuntimeTypes.fnName|21abe108476196d96790b3d40ec1d3889a2e9dc4fdce43241e5543230579627d +fn/PrettyPrinter.RuntimeTypes.typeReference|f734f6d47fca616c1926b13d09a23efb5d620b5a214f096f1c3a36e51ba8f2c8 +fn/Stdlib.HttpClient.request|dc2c6f30499e790d516015edc4d3664ae806960bae0d11148e3bb2a3bfcd9e60 +fn/Stdlib.HttpClient.stream|d6ecec7169b4515ffc41f4e26e6164b362466fb34b2976c6cff2bcb4072c736b +type/Cli.ExecutionError.ExecutionError|e9efb8b1850160b0eaaaa51fa5a0658cc8090c52bef576fe5cf822f863b023ec +type/Cli.ExecutionError.PermissionDenied|1eb2bddcaab09f01c5b050cd44c12eecc3f11764f301ce472efacec55579c1f2 +type/Cli.ExecutionError.Unhandled|aefa4db626bbf85788e5896f4759f5582746137940d41282aabddf8f004403b5 +type/Cli.Scripts.Script|dae5fc6163a0ebe22c6ef1f55ec7b77d455041a914e6364191eec0f214644c88 +type/DarkPackages.Stats|9d63bb09dcae3852c23cda15e84803477a5002d6be54a6d106aec9bf9d5ec6c7 +type/LanguageTools.AtRestTypeChecker.AmbiguousSubject|557a52c2e6050ce51e81e7300b252708c09907e99afed237ffa7eac4c1275f83 +type/LanguageTools.AtRestTypeChecker.Context|17e12adf761418d93bd727e9cafe37787e42897a920a5acaf1f2d9a175da9137 +type/LanguageTools.AtRestTypeChecker.DuplicateSite|964cbef482ab0ba87a061926f3a7c1aa33b16275624c30040094ca62be41eabe +type/LanguageTools.AtRestTypeChecker.Issue|f349e4beb6f2f61882b89ceaa86617adf6cd6f1f52e41308497f86061df40e06 +type/LanguageTools.AtRestTypeChecker.IssueCode|72b4b5fdffaa9c8c82cd273e2483f8fd68d47e765f881b8c20bd7df08bf98acd +type/LanguageTools.AtRestTypeChecker.ItemReport|1ab924b41a1c80b6a72e7d7c4f8feaa13467d5cb0c1c5c8d121597705e1f6208 +type/LanguageTools.AtRestTypeChecker.NameRef|024f7978c58b0080920438aae4c893b9647f1f01b64079d72a2ea6dba44f4542 +type/LanguageTools.AtRestTypeChecker.Report|e52f47898b9038e701d031e122c7ac731158aa2d1656c253f493ada50ffdd9b3 +type/LanguageTools.AtRestTypeChecker.Site|8ee5e516873622675089d638224913ff15b9e46adf70b284c0996b2b43ab70bf +type/LanguageTools.AtRestTypeChecker.StaticType|553b72aa15f3bbd26eb0d7b28281053b95d47cf74a3cb8d05f62a40214d4239b +type/LanguageTools.AtRestTypeChecker.UntrustedBuiltin|aefe439cf486f046d864c03e5b34442bd01fd85b3f77d8045abeaea099757bd7 +type/LanguageTools.AtRestTypeChecker.Verdict|c17cea9c7be860439544eb6d3eb33d378e825030a50ba40407f1f56a5b1b3c30 +type/LanguageTools.BuiltinFunction|f85a8fe29057d6cece428f186740cc5bacdb8d4856c70d09bf07c1078fbb75b9 +type/LanguageTools.BuiltinFunctionParameter|41fe90a5ee71d99032303646efc46d68bbfb7d81a894f0f888b5d252abe7fc4a +type/LanguageTools.BuiltinFunctionPurity|871e4962a7f55a23f35c21043f6759a09aa382972f3b5ca48d279d764695920e +type/LanguageTools.BuiltinValue|a4d157300a079b280f5eeb324b1192550d0eb68b2cf9a92f6b5fa0b46acc0550 +type/LanguageTools.Parser.CliScript.PTCliScriptModule|6081179541a51989bd857570ea0e3621db192bcf57526bee27c86c9a10f33910 +type/LanguageTools.Parser.CliScript.ParseError|6808e3ffbad45a9484f7d362d5b3f55ed7996f741470253a7178b1e108ce9b0f +type/LanguageTools.Parser.Point|267284360b92e3b7af64acd4bead9f7b11699b5df9b51c29e8c62ea702fbdcd0 +type/LanguageTools.Parser.Range|a6ba33909336f9302fcd6d9df47c9be9dc4f85ffd2cb08509d9c991287aa2d97 +type/LanguageTools.Permissions.ApprovalFailure|703e5f0bd3e7527b0295e1b5f797b5413a0d4b2e5a467d5e3e390d56701cff1d +type/LanguageTools.Permissions.Effect|7afb64a44090ba40d3c03f7a75175507007e6c0c4305b24d35badbea8b2e9444 +type/LanguageTools.Permissions.HostRule|cccfab0b6458e3c8e70ffc9849288ca2ea99b63133f87c0293b2b0d46c73af6d +type/LanguageTools.Permissions.HttpRule|d970fa7107d29a7b7edee09daeef0a0a2dfa1bee01e5b81ea69e0fb9d4e22fcc +type/LanguageTools.Permissions.Policy|a44fd0606df64bf8eb9caefddac00dd2a99e7793494b02fa34389bcc79c59514 +type/LanguageTools.Permissions.ProcessRule|17339a7f671e8ba31f33c33fdfb8e6e61a24357d984b8d7af90a1e239d1a808d +type/LanguageTools.Permissions.Rule|9e050e7ba9b5ca5f8080aef78893b89cb8ced303b9a73a63f66e045162e8ea4c +type/LanguageTools.Permissions.Scope|c070e01ea190d8013836dae8b361eaec47f0faff1f0086a9b30ad2c52f62ba1a +type/LanguageTools.ProgramTypes.BinaryOperation|94a439540f7cd066316843e3124648fec75d8e650b7256bb08d428ec0f9d1b5e +type/LanguageTools.ProgramTypes.BranchEventKind|40eb9edeb4d3cd4943f77a9a5491459571949aed52eaa8f9200cffdcd1686290 +type/LanguageTools.ProgramTypes.DB|f86fa6609526793025bff7d6028d83db5126e5c5b32bd9a1cde2526465751c65 +type/LanguageTools.ProgramTypes.DecisionKind|2251b3ac77f0db7a0a7bc6d3b2b85a8ea988b9d8033f11c8444a3d69f2b5790a +type/LanguageTools.ProgramTypes.Deprecation|47b77911f56bc81a6a82dfe81c07d11c88d132efdbef9c4dc8e8807f23de59b2 +type/LanguageTools.ProgramTypes.DeprecationKind|3513452dffa50fed6f0b9273aaf44dbda3e3aa166d9be38f84fba9a93d449ac2 +type/LanguageTools.ProgramTypes.DocPart|edaec509f09809494c45a1122074996158f00ad03c13865eb9cc18cc8647e7b2 +type/LanguageTools.ProgramTypes.Expr|51d1c6d465118e886efbf45f4a6abaa1282b52fc159df96cea2976cec73676ea +type/LanguageTools.ProgramTypes.FQFnName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.ProgramTypes.FQFnName.FQFnName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.ProgramTypes.FQTypeName.FQTypeName|7369292d3e4858b74948d8fab5915d9c3ae77deea0b6fd6b5d7463331142d0b9 +type/LanguageTools.ProgramTypes.FQTypeName.Package|e5d0c6269fdf0c50210733af27f0fed1b5ce593ac2830f0f4b62b723d2a1d2c6 +type/LanguageTools.ProgramTypes.FQValueName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.ProgramTypes.FQValueName.FQValueName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.ProgramTypes.Hash|596e678425be94c61f2bab9d1dd380691cd415167b4b9ed766af4df8ab0ccc2f +type/LanguageTools.ProgramTypes.Infix|8e630370f404b2345b6bb76e6163a1fe5c3ef7c379319d3e38ed5b80dc1e42df +type/LanguageTools.ProgramTypes.InfixFnName|85cdca8d06c978f066f0aaa59b920f53d2f16e2359d1021aebdd44d3b8a08523 +type/LanguageTools.ProgramTypes.ItemKind|642ec69245ff338e71dae980d8c7b1cac623b6b49fdd92375df3f90d06cc862f +type/LanguageTools.ProgramTypes.LetPattern|f957b698ed40cc0c0f4c38159ee7732206d87c84440008c91a5b90b318ccf5a3 +type/LanguageTools.ProgramTypes.LocatedItem|2921046a667c6002f86a7f94bd710bf693f9273e0e8daad8580925af13cab173 +type/LanguageTools.ProgramTypes.MatchCase|7949491033140cd482b2b2aa87c7c8428d6597a02c2dde9abe9152579651c9bc +type/LanguageTools.ProgramTypes.MatchPattern|7b53dddbddbf377ed473e774b41653e1bdca1fd12e667374c37e01bbed8acde6 +type/LanguageTools.ProgramTypes.NameResolution|f0455b86c93c628fad3c05a12096b0d0bdf952dced2cf66716f390dc2673c95a +type/LanguageTools.ProgramTypes.NameResolutionError|f4d41a91096f025fe7d0db4123ef4d31bf47a0602c55da47036f0d5640434f53 +type/LanguageTools.ProgramTypes.PackageFn.PackageFn|0cc514f99b65b8c6ca917880ae86966ad4fcfd256d59940fa9eb606317a3357a +type/LanguageTools.ProgramTypes.PackageFn.Parameter|179565d8649ec92edf9652d1acbc5640890871813c9b29e4298b58e3522ade75 +type/LanguageTools.ProgramTypes.PackageLocation|952f921bae69c1df53ffde67755738bd3071fa97165be6d1e30f1d8c3ee9c706 +type/LanguageTools.ProgramTypes.PackageOp|c5d23318efdf091d000c85b6fec0825d7c5166a39ec156877bb8f3cafe86bce0 +type/LanguageTools.ProgramTypes.PackageType.PackageType|deef75c738b32ba37d01397948c3413cc176f029c7db71323100c352c6699690 +type/LanguageTools.ProgramTypes.PackageValue.PackageValue|ba279c227b51e5fee8ba5c11be7861827b8c6a4ae8f8430407fb743d2e57ffe7 +type/LanguageTools.ProgramTypes.PipeExpr|087d322b1a7e063842a2d0722f1006284a8ff29fbeb90bc0017ed4a21f1b1488 +type/LanguageTools.ProgramTypes.PropagateRepoint|1f8f3258922f4e484c83bd72b003193ae5e43b7123472d3d1d2eb1e441087b8a +type/LanguageTools.ProgramTypes.PropagationPolicy|851ca4c25f20f9c92552ae5e997b9a6c95c0df415ea886bac754e9c820814cf9 +type/LanguageTools.ProgramTypes.Reference|c08a67fb0309f39d09ffce5ec91c05c2c4e0ad99318be2bb9d4bf3e070dd2289 +type/LanguageTools.ProgramTypes.ResolvedName|0951d47e4d55de37859945115cb9bc9be5be53a4bf7516f09c7bb2b6dd6cafaf +type/LanguageTools.ProgramTypes.Search.EntityType|7824e28c37f8b4f994c3f3752e682fb58229e66743c83ae7a6e4f29515cb4f6a +type/LanguageTools.ProgramTypes.Search.SearchDepth|364316241bbd776a630ac9c3f92927d3a634ef4446b9d45258a8436a254781ec +type/LanguageTools.ProgramTypes.Search.SearchQuery|8f81372f03d7148e91328db2cef2d0d85100ddf8ed4042dfc5e3f598c85b5e46 +type/LanguageTools.ProgramTypes.Search.SearchResults|751f8f7eebb10ce34c92278e99a9686bc43a476a448f467eeed11d0042d1eb44 +type/LanguageTools.ProgramTypes.StringSegment|58e0c47bb114d85fad882c54475952c9f4b355279b455e8043f2a2fc73b44218 +type/LanguageTools.ProgramTypes.TypeDeclaration.Definition|003a2de1f7448255e6fd849823be6e80ddb4942199e6b18c72af541bf4dd823c +type/LanguageTools.ProgramTypes.TypeDeclaration.EnumCase|3fe93214c933cdb751e62fd53c1dbeac17c8ffca4d09112babcadbf93c7590a1 +type/LanguageTools.ProgramTypes.TypeDeclaration.EnumField|62c877e2401adbd4328c46f113b6d00e7d17461e281f6f43869212de243831bc +type/LanguageTools.ProgramTypes.TypeDeclaration.RecordField|179565d8649ec92edf9652d1acbc5640890871813c9b29e4298b58e3522ade75 +type/LanguageTools.ProgramTypes.TypeDeclaration.TypeDeclaration|068e22041cea13c341085e83de7d4d94211b77cee6ff034a9df4f9c11d47a0d8 +type/LanguageTools.ProgramTypes.TypeReference|d3c3e692deee3fd94d133ae4a92e5754c8adf0bcd563142a10999d39b4b36962 +type/LanguageTools.RuntimeTypes.Applicable|6cca0bb6ce4f7fed1fe32e5ad57b1ee738706d644114cb4bbe5e11a4feae9b38 +type/LanguageTools.RuntimeTypes.ApplicableLambda|b95064f7830ecd768363dd3d2664244081d12fe421a694e2c90be9eb40e3683d +type/LanguageTools.RuntimeTypes.ApplicableNamedFn|06b91d3d07a600b573f8bd28fbbcdc4e662b520e5077080efe3dcaa7bdf63f58 +type/LanguageTools.RuntimeTypes.Dval|f3cf95edd1bfe13a32c69120d061d8677b22eb8897985988f26a5c8e4011aa55 +type/LanguageTools.RuntimeTypes.FQFnName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.RuntimeTypes.FQFnName.FQFnName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.RuntimeTypes.FQTypeName.FQTypeName|7369292d3e4858b74948d8fab5915d9c3ae77deea0b6fd6b5d7463331142d0b9 +type/LanguageTools.RuntimeTypes.FQTypeName.Package|e5d0c6269fdf0c50210733af27f0fed1b5ce593ac2830f0f4b62b723d2a1d2c6 +type/LanguageTools.RuntimeTypes.FQValueName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.RuntimeTypes.FQValueName.FQValueName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.RuntimeTypes.Hash|596e678425be94c61f2bab9d1dd380691cd415167b4b9ed766af4df8ab0ccc2f +type/LanguageTools.RuntimeTypes.Instruction|2ddd25bc868e04b2f502f22edbefe4318a1e4698b187d3b1a9f7caf2dcec8fa2 +type/LanguageTools.RuntimeTypes.Instructions|cbf1ab8b258406546f4a879cdfeec802854af3cba80983d49bd856d45face991 +type/LanguageTools.RuntimeTypes.KnownType|096e7ddd17ad9900e2905265353ada42982d0f1a35dfa39da78751aa5d8bbdaa +type/LanguageTools.RuntimeTypes.LambdaImpl|b140c92e6e43ebcb68a6959347769de5b641c7149ecaa7c702ad5536ff5d2147 +type/LanguageTools.RuntimeTypes.LetPattern|f8b3270bed302c8e06dcb7b8a5e932ce26b404b2167d1d3109158066852c6702 +type/LanguageTools.RuntimeTypes.MatchPattern|c32d33ebbe9cb02c0447b2799e4acfa285325e84c024f9717d9bd439b57e4d89 +type/LanguageTools.RuntimeTypes.NameResolution|3abcaf537e21e80c102fcc20ee62566341c528426e7c7f064b6136f6c88eaba0 +type/LanguageTools.RuntimeTypes.NameResolutionError|f4d41a91096f025fe7d0db4123ef4d31bf47a0602c55da47036f0d5640434f53 +type/LanguageTools.RuntimeTypes.RuntimeError.Applications.Error|f6b584457e9e5df4af217353347c018df6c9f022b412296150ac08387ea2bd60 +type/LanguageTools.RuntimeTypes.RuntimeError.Bools.Error|ebbfa65c0688980f9ee83311f7cea51e567bd74fabb180302826a51f8c5639c2 +type/LanguageTools.RuntimeTypes.RuntimeError.CLIs.Error|da3f52e13faf9cb8c060d9bde7ef7f26d46c45af91292d4ee41ff732ee40f68f +type/LanguageTools.RuntimeTypes.RuntimeError.Dicts.Error|29c76bcdde500809134fdd9e9cc25553c2bf410b7592c29a695a08c0399c5162 +type/LanguageTools.RuntimeTypes.RuntimeError.Enums.Error|cf5599570dad0c6653cb791ec3c3c8489cdc394667f3c7ca6a8fc9493966191e +type/LanguageTools.RuntimeTypes.RuntimeError.Error|872d08a657bc317393453c9bf5ad4f6ad5763741076137054f3d91fc0aa9b825 +type/LanguageTools.RuntimeTypes.RuntimeError.Ints.Error|a3c5852ebe977e62763b25487608bce2ed1903a832e035993c99fb45b5a56fdf +type/LanguageTools.RuntimeTypes.RuntimeError.Jsons.Error|d43d739fab1d85693d4dced787167c1c05ffae26873d6a683c3b96a2fb19726b +type/LanguageTools.RuntimeTypes.RuntimeError.Lets.Error|2befad20e7f6306338a5c730921d9d031d6b4b0b1785fa6443e18ca10c74f067 +type/LanguageTools.RuntimeTypes.RuntimeError.Lists.Error|a698587423995026c5e61cdbaeb2894a94e81a0a88bebe3da1a87a84274f151c +type/LanguageTools.RuntimeTypes.RuntimeError.Matches.Error|4b1e5ad3647a3dc2aa2f7d493b7637335604f5d9566fb3af4f4cca5bfca46d18 +type/LanguageTools.RuntimeTypes.RuntimeError.Records.Error|1a0f2e292b56b519e4d969f413144c16f3ea7cbe73b1056cab8d37c48b26275e +type/LanguageTools.RuntimeTypes.RuntimeError.Statements.Error|4240e113859d32841d783d78e3169f5a8ae8b87c3fe6b169d9c52ec3d2da9650 +type/LanguageTools.RuntimeTypes.RuntimeError.Strings.Error|2182dbf5bf844104dc70780c1fef6866618e0f96a6a0c764716112fb1d4fe07f +type/LanguageTools.RuntimeTypes.RuntimeError.Unwraps.Error|32ef16d276d78a349fabbbe2f7872b9aa85ba6742f25dbbdb5b2f0477e55cfaa +type/LanguageTools.RuntimeTypes.StringSegment|9b4999992350c36a25dcd53a3fb0c92dc6449c534713b392c74bf2c3cab989e2 +type/LanguageTools.RuntimeTypes.TypeReference|611535d15626223e1245fa4023f81691430a58963c5e0410bf52659614ec8496 +type/LanguageTools.RuntimeTypes.ValueType|98dd5d3be603f70536815aca3ae3db82a5788336f3254237e9d5e48bdd2695f2 +type/LanguageTools.Sign|c8276aff8c52a06bbfacac24d654f3bc3d4a2a37bb0b9ba0ef98d2b99ca56dca +type/LanguageTools.WrittenTypes.BinaryOperation|94a439540f7cd066316843e3124648fec75d8e650b7256bb08d428ec0f9d1b5e +type/LanguageTools.WrittenTypes.Expr|72db9e5ad59e416e141cfee8075f6b27cd44a2b11e315a508951411a667339f4 +type/LanguageTools.WrittenTypes.FnDeclaration.FnDeclaration|3cf6af515e205f67127fd5725bca0dd4f0b1e07ce234acd3d490c826d613cf8e +type/LanguageTools.WrittenTypes.FnDeclaration.NormalParameter|2dcf9acdca3ca0abeeddb4d7cfe65195dbaa17cfb3cde417f4fd8ddb673bc8fa +type/LanguageTools.WrittenTypes.FnDeclaration.Parameter|bf0f75061470b8faa1494e67e3c4f0a25ae3d2f3b91542265f938a5db7cd64ea +type/LanguageTools.WrittenTypes.FnDeclaration.UnitParameter|7b1f2b8d5d41696eb8a33a3b77fcad6287a4af6ef4d3e9da98a9aa4512ba5e91 +type/LanguageTools.WrittenTypes.FnIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.Infix|8e630370f404b2345b6bb76e6163a1fe5c3ef7c379319d3e38ed5b80dc1e42df +type/LanguageTools.WrittenTypes.InfixFnName|85cdca8d06c978f066f0aaa59b920f53d2f16e2359d1021aebdd44d3b8a08523 +type/LanguageTools.WrittenTypes.LetPattern|a01a8aee1ae846263a89c26bd4bedb013403069beaf1bc6b4bb7731e7c2375c1 +type/LanguageTools.WrittenTypes.MatchCase|f05617d9c57f11337cafaeb401d4da7504870bc0a634b24776de9e530b5144fd +type/LanguageTools.WrittenTypes.MatchPattern|528c1badff3131b6910b03a3091b0ebe465874dbd9dac928b63f25ec5f1f8b02 +type/LanguageTools.WrittenTypes.ModuleDeclaration.Declaration|c89618597f1c0dad4b0cfa53ca652505826a409931728eb89db3f39b5c38ac22 +type/LanguageTools.WrittenTypes.ModuleDeclaration.ModuleDeclaration|1a418dd008179924b912513c94be30868d2eb4919a039c64a6a364a68c554a61 +type/LanguageTools.WrittenTypes.ModuleIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.ParsedFile|07382e181d7d13690ef13b4f95f2ba8b07e7140332ce2af981d7a7296ba64e99 +type/LanguageTools.WrittenTypes.PipeExpr|c885f6f5219f56d346406b636d0d7948517f8f2cdb0f80135707b23d41eeac70 +type/LanguageTools.WrittenTypes.QualifiedFnIdentifier|0f68e49a14653866fe015d68714149c50cd572ab8aaf9f5c0e66cece4e4f3856 +type/LanguageTools.WrittenTypes.QualifiedTypeIdentifier|e813835428f56f4a33a22ca382385c82509d255644639bae0daec48f94c26398 +type/LanguageTools.WrittenTypes.SourceFile.SourceFile|630e8cc6e7bed1a47eac5add8a5cdc1c2f3e5cf41ed220a258710f29669f4554 +type/LanguageTools.WrittenTypes.SourceFile.SourceFileDeclaration|53a59a7e8a6a4e670834cb26ca70aa8475fc401aa447e2e85f32d434fd98f5f9 +type/LanguageTools.WrittenTypes.StringSegment|3b069928919b9a8ea9df0efc533b908daded5e5f4650c5886aa5fd319a2849eb +type/LanguageTools.WrittenTypes.TypeDeclaration.Definition|240935955526e9e0c62f6901bbd25bb1a4887596cd0c51d5382f6fc42ba4d732 +type/LanguageTools.WrittenTypes.TypeDeclaration.EnumCase|89ece486356b4be56caabfabbddcf7cc4eea6403a3510c2f16855a6708d094fa +type/LanguageTools.WrittenTypes.TypeDeclaration.EnumField|9b73a8dfadc90d77e9b5707318ab34d659c8cd9a66e6cabe96a8ed425bb86a3a +type/LanguageTools.WrittenTypes.TypeDeclaration.RecordField|e57cc695dd2e30a35ec8351b28b14f8e7b71ec9be5466e9af3096ce0f9ea49af +type/LanguageTools.WrittenTypes.TypeDeclaration.TypeDeclaration|ec050095398e9b4d8e564c269a9855981401110218de1c340bfdb6f6c8647d05 +type/LanguageTools.WrittenTypes.TypeIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.TypeReference.Builtin|e1a6e53933d0ba4d11f08c592bc7b5c997c791c614e9451ddb9278257bc529f7 +type/LanguageTools.WrittenTypes.TypeReference.TypeReference|df116cfc0b6f7d492401ec365cfec5aa580a31a3486731f4d99bf86114b2af6a +type/LanguageTools.WrittenTypes.ValueDeclaration.ValueDeclaration|74051a4d9394cf0bed20c5fa0eeee6603a1aa0d55eba59ca57c6a83ef0cdfbf9 +type/LanguageTools.WrittenTypes.ValueIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.VariableIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/PrettyPrinter.RuntimeTypes.RuntimeError.ErrorMessage|2314408c9017bab823414db19433005893a4d09e6235a41011e3ab1f5ae79818 +type/Stdlib.AltJson.Json|a3b6038ebf54eb71804f0ad7b96b883e442ccf4d01af8ac220c6981fa3530305 +type/Stdlib.AltJson.ParseError.ParseError|8dc77b20b13969bee07766b01d9afa75c78fbfe780443c621742fa2021831abe +type/Stdlib.Cli.ExecutionOutcome|1a4a0d68fa0df09a91283de7e194709cdd210793d8e517b44e26090caeaf031a +type/Stdlib.Cli.FileSystem.FileError|76b5a147c2369e90d08ae51df41920e46e38c0d5b5828539542049cbc63bd188 +type/Stdlib.Cli.OS.OS|b7c0e15d61d6652c5be3cdc9360637524332b72aec228c3a771e2f3f88a8a1fe +type/Stdlib.Cli.Posix.Error|a55af955adffd4fd5aa31b9c0f097a496a161b9514a18922a54943b95aaec4b0 +type/Stdlib.Cli.Stdin.Key.Key|788e35c44bf7b0546cfce602a673dcc38ee2a2e84680e6e8df39f5c281325085 +type/Stdlib.Cli.Stdin.KeyRead.KeyRead|12da87b8fc16efa28f98572b629ca38be3729f94d928e2c60e17242d03f11f9d +type/Stdlib.Cli.Stdin.Modifiers.Modifiers|5473100a8e7309f3438b5ef92a99fba6518d937b3de96d2c3e4169a21e097100 +type/Stdlib.Float.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Stdlib.Http.Request|e8c423f68d1d0743b84b7ac20b131372c646a5ce534c8438c8801d18e6274362 +type/Stdlib.Http.Response|4d686467d89ac2a7af562f68fc4d9048d12774ffd4827a6dde5efd69cc127a74 +type/Stdlib.HttpClient.BadHeader|dbfe4487bd474fdf0ffccaf4d8f546785e1037becc1f5527696e053e5199f1b8 +type/Stdlib.HttpClient.BadUrlDetails|1dbb355daa5a3c3531e71a0682e0aa20ccf6751d136abcfdcca5def779c21eb5 +type/Stdlib.HttpClient.RequestError|4e899f22b33061bc75ced7d558833eb454908a5ea4f414980c31c778289dcfcf +type/Stdlib.HttpClient.Response|4d686467d89ac2a7af562f68fc4d9048d12774ffd4827a6dde5efd69cc127a74 +type/Stdlib.HttpClient.StreamResponse|9a509af8b0b16226d61a9f4bea0ffa533c36a889606c161799398b92c22dab78 +type/Stdlib.Int.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Stdlib.Int128.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int16.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int32.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int64.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int8.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Json.ParseError.JsonPath.Part.Part|a19fcef1b58b4010b122abe9bef2401d38085229b44a085eb27e35f60652f3dc +type/Stdlib.Json.ParseError.ParseError|7e14f06f970c01da29e51d928202774b414e098b0376f49bc2234a8b857fae5f +type/Stdlib.Option.Option|c58cf283d0e5d6634a76fa30ea5b0d29e615e1c4fe1cc2bfca27077cd72bc072 +type/Stdlib.Result.Result|03549ecc7eb39c974d3f7181c87a3ee9337fc91307ecb2c24351b0df4809ce15 +type/Stdlib.Sqlite.Value|41c352765ba7273fab25c56327f91a554550dc3db8e1417a6fcf2e4da5e586e7 +type/Stdlib.UInt128.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt16.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt32.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt64.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt8.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Uuid.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Tracing.FnCall|a117302204173443b563b8ae07857467463bd14c77bec3cf12653bab5a4b04ce +type/Tracing.InputVar|e75e7d0c6e9202b9ec72823bdb3dbc668a22d61d4b3468dd3f1e729b814d461d +type/Tracing.TraceData|542e314e375133a017fb9f27733a0ede6a167af4d70edf1c64aaa9aa0145c76e +type/Tracing.TraceSummary|d7623097e2da2eb7216d8b77fa6877d586ed369da09b5bea50eca6c5576b5a6f diff --git a/scripts/build/check-seed-carries-refs b/scripts/build/check-seed-carries-refs index 7b5dabb60d..c875bd92bc 100755 --- a/scripts/build/check-seed-carries-refs +++ b/scripts/build/check-seed-carries-refs @@ -33,6 +33,8 @@ def main() -> int: if "|" in line} ) if not refs: + # Only a fresh clone before its first reload, where MSBuild has touched an empty one. + # The file is tracked, so in every other case there is something to check. print("check-seed-carries-refs: skipped (package-ref-hashes.txt is empty)") return 0 From bc334141c13193b63d53d1b9d315cdd09ef4b2ba Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 15:38:59 -0400 Subject: [PATCH 18/32] Resolve type refs from the store by name, as fns already do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 189 of the 206 pinned refs are types, so the 17 fns that resolved by name were the small half of the coupling between the kernel and the package set. What made this look impossible was a claim in `PackageRefs` that turned out to be half right: a `DRecord` the kernel builds does carry its type's hash, so a store whose version of that type has a different SHAPE would hand Dark a value it cannot typecheck. That is what the declaration check is for. What the claim missed is which way the coupling runs. All 44 `fromDT` conversions match `DEnum(_, _, …)` -- F# never reads the type name it receives, only the one it writes. So the hash is write-only, used to tag values rather than recognise them, which makes it a lookup by name that happens to be cached in a file rather than a contract that has to be. The net is the fn net with the comparison swapped: shapes instead of signatures, descriptions stripped so a doc edit does not cost the store its binding. The case it exists for is the one with NO pin -- a type a branch has just authored, which F# on the same git branch wants to reference. There is nothing to compare against, so the store's answer is taken. Caching is not a nicety. These sit under Option and Result construction, so a raw query per call measured at 46% more allocation on the reference workload (9.4 -> 13.6 MB) with the wall clock barely moving, which is why time would not have caught it. Two layers: `Caching.withCache` for the store lookup, cleared by the fold, and a memo in the ref closure against a generation the fold bumps, so a resolved ref is one int compare. `record` stays off the hit path; it is a `Map.add` into the generator's lookup table and calling it per resolution allocates a map node per Option construction. Lands at about 6.5% over, isolated by running the same workload with `DARK_REFS_BY_NAME=0`. WHAT THIS DOES NOT DO, and it is the reason the arc is not finished: the resolver reads `locations`, which is main's projection and has no branch column. Authoring two items on a branch leaves `locations` with zero rows for them and `op_branches` with two. So a type authored on a branch is invisible here, and the coordination case this exists for does not work yet. The generator has the same blind spot, for the same reason. Both are named in the design note. --- backend/src/LibDB/PackageManager.fs | 122 ++++++++++++++++++++++-- backend/src/LibExecution/PackageRefs.fs | 89 +++++++++++++---- 2 files changed, 185 insertions(+), 26 deletions(-) diff --git a/backend/src/LibDB/PackageManager.fs b/backend/src/LibDB/PackageManager.fs index 6f55fbd4f7..9aade47bc0 100644 --- a/backend/src/LibDB/PackageManager.fs +++ b/backend/src/LibDB/PackageManager.fs @@ -663,6 +663,33 @@ let private sameSignature /// the only way back is an environment variable you have to know exists. With it, a rebind whose /// signature does not match what this build compiles against is refused, loudly, and the pinned /// version is used: a wrong edit degrades rather than bricks. +/// What `Darklang..` of this kind binds to on main, CACHED. +/// +/// Through `Caching.withCache`, which the fold clears, so this expires exactly when a rebinding +/// could have changed the answer. Caching is not a nicety here. These sit under Option and Result +/// construction, so they run constantly, and a raw query per call measured at **46% more +/// allocation on the reference workload** -- 9.4 MB to 13.6 MB -- with the wall clock barely +/// moving, which is why time would not have caught it. +/// +/// `withCache` does not cache `None`, which is right: a name the store does not bind yet may bind +/// later, and a cache has no way to hear about that. +let private kernelHashByName = + Caching.withCache + (fun ((itemType : string), (modulesStr : string), (name : string)) -> + uply { + return! + Sql.query + $"""SELECT item_hash + FROM locations + WHERE owner = 'Darklang' AND modules = @modules AND name = @name + AND item_type = '{itemType}' AND unlisted_at IS NULL AND source != 'unbind' + LIMIT 1""" + |> Sql.parameters + [ "modules", Sql.string modulesStr; "name", Sql.string name ] + |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") + }) + + let private resolveKernelFnByName (modules : string list) (name : string) @@ -671,15 +698,7 @@ let private resolveKernelFnByName let modulesStr = String.concat "." modules let candidateHash = - Sql.query - "SELECT item_hash - FROM locations - WHERE owner = 'Darklang' AND modules = @modules AND name = @name - AND item_type = 'fn' AND unlisted_at IS NULL AND source != 'unbind' - LIMIT 1" - |> Sql.parameters [ "modules", Sql.string modulesStr; "name", Sql.string name ] - |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") - |> fun t -> t.Result + (kernelHashByName ("fn", modulesStr, name) |> Ply.toTask).Result match candidateHash with | None -> None @@ -720,6 +739,91 @@ let private resolveKernelFnByName LibExecution.PackageRefs.resolveFnByName <- resolveKernelFnByName +/// Do two type declarations have the same SHAPE? +/// +/// What has to hold is what a value carries: the kind of declaration, the fields or cases in +/// order with their types, and how many type parameters. Descriptions are the author's business, +/// so they are stripped before comparing -- a doc edit must not cost the store its binding. +let private sameDeclaration + (expected : PT.PackageType.PackageType) + (candidate : PT.PackageType.PackageType) + : bool = + let stripDefinition (d : PT.TypeDeclaration.Definition) = + match d with + | PT.TypeDeclaration.Alias t -> PT.TypeDeclaration.Alias t + | PT.TypeDeclaration.Record fields -> + fields + |> NEList.map (fun f -> { f with description = "" }) + |> PT.TypeDeclaration.Record + | PT.TypeDeclaration.Enum cases -> + cases + |> NEList.map (fun c -> + { c with + description = "" + fields = c.fields |> List.map (fun f -> { f with description = "" }) }) + |> PT.TypeDeclaration.Enum + + stripDefinition expected.declaration.definition = stripDefinition + candidate.declaration.definition + && List.length expected.declaration.typeParams = List.length + candidate.declaration.typeParams + + +/// The net for TYPE refs. See `resolveKernelFnByName`; the difference is what gets compared. +/// +/// The case this exists for is the one with NO pin: a type a branch has just authored, which F# +/// on the same git branch wants to reference. There is nothing to compare against, so the store's +/// answer is taken. An existing type whose shape moved falls back to the pin, because that is the +/// shape this build was compiled against and a `DRecord` it builds has to typecheck against it. +let private resolveKernelTypeByName + (modules : string list) + (name : string) + : string option = + try + let modulesStr = String.concat "." modules + + let candidateHash = + (kernelHashByName ("type", modulesStr, name) |> Ply.toTask).Result + + match candidateHash with + | None -> None + | Some candidate -> + match LibExecution.PackageRefs.pinnedTypeHash modules name with + // Nothing pinned: a type this build has never seen, which is the branch case. + | None -> Some candidate + | Some pinned when pinned = candidate -> Some candidate + | Some pinned -> + let typeFor (h : string) = (PMPT.Type.get (PT.Hash h) |> Ply.toTask).Result + + match typeFor pinned, typeFor candidate with + | Some expected, Some actual when sameDeclaration expected actual -> + Some candidate + | Some _, Some _ -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the store binds the type Darklang.{modulesStr}.{name} to a version whose " + + "shape is not the one this build expects, so the built-in version is being used " + + "instead." + ) + None + | _, None -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the type Darklang.{modulesStr}.{name} is bound to content this store does " + + "not have, so the built-in version is being used instead." + ) + None + | None, _ -> + // The PINNED version is missing. Nothing to compare against, and refusing leaves no + // type at all, so trust the store. + Some candidate + with _ -> + None + +LibExecution.PackageRefs.resolveTypeByName <- resolveKernelTypeByName + +// The ref closures memoize what the store said; the fold is when that can stop being true. +Caching.register LibExecution.PackageRefs.invalidateStoreResolution + + let private otherBranchOps = System.Collections.Concurrent.ConcurrentDictionary>() diff --git a/backend/src/LibExecution/PackageRefs.fs b/backend/src/LibExecution/PackageRefs.fs index fba40797aa..b6dfe81189 100644 --- a/backend/src/LibExecution/PackageRefs.fs +++ b/backend/src/LibExecution/PackageRefs.fs @@ -148,21 +148,54 @@ let pinnedFnHash (modules : string list) (name : string) : string option = let fqn = $"""fn/{String.concat "." modules}.{name}""" getHashes () |> Map.tryFind fqn -/// How a FN ref resolves against the live store, when it does. +/// The hash this BUILD pins for a TYPE ref. Same role as `pinnedFnHash`: the declaration this +/// build was compiled against, to compare a candidate rebinding with. +let pinnedTypeHash (modules : string list) (name : string) : string option = + let fqn = $"""type/{String.concat "." modules}.{name}""" + getHashes () |> Map.tryFind fqn + +/// How a ref resolves against the live store, when it does. /// /// Installed by whoever owns the store, because `LibDB` depends on `LibExecution` and not the /// other way round: `PackageRefs` cannot read `locations` itself. `None` until installed, and /// `None` for a name the store does not bind. /// -/// Why fns and not types. F# only CALLS these seventeen; it never takes one apart. So the frozen -/// contract is the name and the signature, and the newest committed binding may win -- which is -/// what makes "the store is the source" true of the CLI's own entry points, rather than true of -/// everything except them. A TYPE is different and must stay pinned: a `DRecord` the kernel -/// builds carries its type's hash, so a store whose newest version of that type has a different -/// shape would hand the kernel a value it cannot read. +/// Both kinds go through the store now, and the net is the same shape for each: compare the +/// candidate against the PINNED version -- signatures for a fn, the declaration for a type -- and +/// fall back to the pin, loudly, on a mismatch. +/// +/// Types took longer to get here because of a claim that turned out to be half right. A `DRecord` +/// the kernel builds does carry its type's hash, so a store whose version of that type has a +/// different SHAPE would hand Dark a value it cannot typecheck. That is what the declaration +/// check is for. What the claim missed is which direction the coupling runs: F# never reads the +/// type name it receives -- all 44 `fromDT` conversions wildcard it -- so the hash is write-only, +/// used to tag values rather than to recognise them. Which makes it a lookup by name that happens +/// to be cached in a file, not a contract that has to be. +/// +/// What this buys, and it is the point: a type a branch has just authored has NO pin to compare +/// against, so it resolves from the store directly. That is what lets F# on a git branch reference +/// package code authored on a dark branch. let mutable resolveFnByName : (string list -> string -> string option) = fun _ _ -> None +/// See `resolveFnByName`. Separate hook because the check differs: a type is compared by its +/// DECLARATION, not by a signature. +let mutable resolveTypeByName : (string list -> string -> string option) = + fun _ _ -> None + +/// Bumped whenever the store could have rebound a name, by whoever owns the store. +/// +/// The ref closures below cache the store's answer against this, which turns a resolved ref into +/// one int compare and no allocation. That is not a micro-optimisation: these sit under Option and +/// Result construction, and asking the store per call measured at 46% more allocation on the +/// reference workload. Caching inside the closure rather than inside the store lookup is what +/// removes the last of it -- a cache one layer down still pays a key allocation per call. +let mutable private storeGeneration = 0 + +/// Drop every ref's memo of what the store said. `LibDB.Caching` calls this on every fold, which +/// is the only moment a name's binding can move. +let invalidateStoreResolution () : unit = storeGeneration <- storeGeneration + 1 + /// ON by default: the store is the source, and these seventeen are the places that matters most. /// Edit the pretty-printer and the binary you already have starts using it. /// @@ -173,6 +206,8 @@ let mutable resolveFnByName : (string list -> string -> string option) = let private byNameEnabled : Lazy = lazy (System.Environment.GetEnvironmentVariable "DARK_REFS_BY_NAME" <> "0") +let private currentStoreGeneration () : int = storeGeneration + /// Shared body of `Type.p` and `Fn.p`: a closure resolving `/.` /// against the hash file. Resolution is cached once per hash generation: the answer /// cannot change while the generation is stable, and resolving per call costs an @@ -188,22 +223,42 @@ let private makeRef : unit -> string = let mutable cachedGen = -1 let mutable cached = "" + // The store's answer, against `storeGeneration` rather than the hash file's: a rebinding moves + // the store without touching the file. `ValueNone` means "asked, and the store had nothing", + // which is worth remembering too -- most refs on a store that does not bind them would otherwise + // pay a lookup per call. + let mutable storeGen = -1 + let mutable storeCached = ValueNone fun () -> - // The store's answer is NOT cached by generation: the generation tracks the hash file, and a - // rebinding moves the store without touching it. Cheap enough -- one indexed lookup, against - // the interpolated key and Map walk the pinned path pays anyway. + // `record` is deliberately NOT on the hit path. It is a `Map.add` into the calling module's + // `_lookup`, so calling it per resolution allocates a map node per Option construction: 0.8 MB + // on the reference workload, measured, which is all of what this path costs. The generator + // only needs to have seen each hash once. let fromStore = - if kind = "fn" && byNameEnabled.Force() then - resolveFnByName modules name + if byNameEnabled.Force() then + let gen = currentStoreGeneration () + if gen = storeGen then + storeCached + else + let answer = + match kind with + | "fn" -> resolveFnByName modules name + | "type" -> resolveTypeByName modules name + | _ -> None + |> ValueOption.ofOption + storeGen <- gen + storeCached <- answer + match answer with + | ValueSome hash -> record hash + | ValueNone -> () + answer else - None + ValueNone match fromStore with - | Some hash -> - record hash - hash - | None -> + | ValueSome hash -> hash + | ValueNone -> let gen = currentGeneration () if gen = cachedGen then From 100fe5fb9a1a0b2ae2ffd9ee0458f14ff060ea29 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:25:13 -0400 Subject: [PATCH 19/32] Make the kernel and the package set check that they agree The two reference each other in both directions -- the kernel names 17 fns and 189 types, the package code calls builtins -- and a kernel and a package set are compatible only if both resolve. Nothing checked either direction, and one of them could not be checked at all. DIRECTION ONE, and three defects in the way. `selectBranch` did not invalidate the caches. A one-shot `dark` selects its branch before resolving anything and never notices; the LSP, the REPL and any daemon would answer for the branch they started on, forever. My bug, from adding the memo earlier today. `PackageRefsGenerator` read `locations`, which is main's projection and has no branch column -- authoring two items on a branch leaves zero rows there and two in `op_branches`. So standing on a branch it wrote a hash file describing main, silently omitting the items the branch exists to add, which is the case where the file matters most: it is what lets somebody else build the F# that references them. `refs generate` in LocalExec never selected the stored branch at all. With those fixed, type refs resolve through the branch overlay. Fns deliberately do not: resolving one through an overlay means the binary runs a branch's parser the moment you stand on that branch, which is not something you opted into by switching. A type declaration is inert -- resolving it decides what a value is tagged with and executes nothing -- so a branch may lend the kernel its types and may not lend it its code. DIRECTION TWO needed a primitive that did not exist. `package_dependencies` is package-to-package only; `PackageItem.fnPackageHash` answers `None` for a builtin, so the call was dropped on the floor and a store could not say which kernel it needed. `package_builtin_deps` records it, filled by the same AST walk that fills the other -- one walk, because walking twice to ask two questions about the same nodes is how the two answers drift apart. That replaces the two builtin discipline tests that grep `.dark` text off disk and evaporate the day `packages/` goes. Strictly better than they were: the real call graph rather than a regex. `refs check` asks both at once, wired into every build after a compile or a reload. At build time and completely, because the ref closures are lazy -- an unresolvable ref is otherwise found whenever some code path happens to reach it, which can be a different day and an unrelated command. The case it exists for is checking out somebody's git branch without their package work. All 206 kernel refs resolve, and all 734 builtins this package set calls exist in this kernel. Both halves verified by breaking them: an F# ref to a type nothing binds, and a row claiming the set calls a builtin that was deleted. An EMPTY `package_builtin_deps` is refused rather than passed -- it is a projection, so a store that got the table from a release step without re-folding would otherwise report success having asked nothing, which is exactly how the tests it replaces would have quietly stopped covering anything. The raise for an unresolvable ref now names the type and the three ways out, instead of saying the hash file is stale, which was usually untrue and never actionable. --- backend/migrations/schema/07-names.sql | 22 ++++ backend/src/LibDB/DependencyExtractor.fs | 47 +++++++- backend/src/LibDB/PackageManager.fs | 86 +++++++++++++- backend/src/LibDB/PackageOpPlayback.fs | 40 +++++++ backend/src/LibDB/PackageRefsGenerator.fs | 11 +- backend/src/LibDB/Purge.fs | 5 + backend/src/LibDB/Releases.fs | 19 +++ backend/src/LibDB/Seed.fs | 2 + backend/src/LibExecution/PackageRefs.fs | 60 +++++++++- backend/src/LocalExec/LocalExec.fs | 121 ++++++++++++++++++++ backend/tests/Tests/OpsProjections.Tests.fs | 3 +- scripts/build/_buildplan.py | 8 ++ scripts/build/compile | 19 +++ scripts/testing/test-build-planning.py | 9 +- 14 files changed, 437 insertions(+), 15 deletions(-) diff --git a/backend/migrations/schema/07-names.sql b/backend/migrations/schema/07-names.sql index 68078e4b59..ec6925c89c 100644 --- a/backend/migrations/schema/07-names.sql +++ b/backend/migrations/schema/07-names.sql @@ -112,6 +112,28 @@ CREATE TABLE IF NOT EXISTS package_dependencies ( ); CREATE INDEX IF NOT EXISTS idx_package_dependencies_depends_on ON package_dependencies(depends_on_hash); + +-- Which BUILTINS an item's body calls. A separate table from `package_dependencies` because a +-- builtin edge is a different kind of thing: a builtin is not content-addressed, it is a (name, +-- version) in whatever kernel you are running, so there is no hash to join on and none of the +-- location columns apply. +-- +-- This is what lets a store say what KERNEL it needs. Without it you can ask which package items +-- reference each other and not which builtins they call, so half of the kernel/package-set +-- interface is invisible and a builtin can be deleted out from under code that calls it. The +-- checks that used to answer this grepped `.dark` text off disk, which stops being possible the +-- day packages come from a seed rather than a tree. +-- +-- Derived, like `package_dependencies`: rebuilt by the fold, dropped by `Seed.export`. +CREATE TABLE IF NOT EXISTS package_builtin_deps ( + item_hash TEXT NOT NULL, + builtin_name TEXT NOT NULL, + builtin_version INTEGER NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_package_builtin_deps_unique + ON package_builtin_deps(item_hash, builtin_name, builtin_version); +CREATE INDEX IF NOT EXISTS idx_package_builtin_deps_name + ON package_builtin_deps(builtin_name); CREATE INDEX IF NOT EXISTS idx_package_dependencies_item ON package_dependencies(item_hash); -- Partial index for the propagation query: "who depends on this diff --git a/backend/src/LibDB/DependencyExtractor.fs b/backend/src/LibDB/DependencyExtractor.fs index 4c8ef0158f..9534a37bea 100644 --- a/backend/src/LibDB/DependencyExtractor.fs +++ b/backend/src/LibDB/DependencyExtractor.fs @@ -43,9 +43,19 @@ type private Work = | MatchCase of PT.MatchCase | PipeExpr of PT.PipeExpr -let private extract (roots : List) : List = +/// A BUILTIN this item's body calls. Not a `Dependency`: a builtin is not content-addressed, it is +/// a (name, version) in whatever kernel you are running, so there is no hash to depend ON. +/// +/// Collected by the same walk, because walking the AST twice to ask two questions about the same +/// nodes is how the two answers drift apart. +type BuiltinDependency = { name : string; version : int } + +let private extract + (roots : List) + : List * List = let work = System.Collections.Generic.Stack() let mutable dependencies : List = [] + let mutable builtins : List = [] let pushInOrder (items : List) : unit = items |> List.rev |> List.iter work.Push @@ -236,6 +246,17 @@ let private extract (roots : List) : List = | PT.EFnName(_, nr) -> addNameResolution nr PT.ItemKind.Fn PackageItem.fnPackageHash + // `fnPackageHash` answers `None` for a builtin, so the package walk drops it. This is the + // only place the call is visible, and it is what a store needs to be able to say which + // kernel it requires. + match nr.resolved with + | Ok resolved -> + match resolved.name with + | PT.FQFnName.Builtin b -> + builtins <- { name = b.name; version = b.version } :: builtins + | PT.FQFnName.Package _ -> () + | Error _ -> () + | PT.ELambda(_, _, body) -> work.Push(Expr body) | PT.EInfix(_, _, lhs, rhs) -> @@ -265,11 +286,11 @@ let private extract (roots : List) : List = work.Push(Expr next) work.Push(Expr first) - List.rev dependencies + (List.rev dependencies, List.rev builtins |> List.distinct) /// Extract all references from an expression without recursive stack use. -let extractFromExpr (expr : PT.Expr) : List = extract [ Expr expr ] +let extractFromExpr (expr : PT.Expr) : List = fst (extract [ Expr expr ]) /// Extract all references from a function definition @@ -282,6 +303,7 @@ let extractFromFn (fn : PT.PackageFn.PackageFn) : List = |> List.map (fun parameter -> TypeRef parameter.typ)) @ [ TypeRef fn.returnType ] ) + |> fst |> List.distinct @@ -294,12 +316,13 @@ let extractFromFnSignature (fn : PT.PackageFn.PackageFn) : List = |> List.map (fun parameter -> TypeRef parameter.typ)) @ [ TypeRef fn.returnType ] ) + |> fst |> List.distinct /// Extract all references from a value definition let extractFromValue (value : PT.PackageValue.PackageValue) : List = - extract [ Expr value.body ] |> List.distinct + extract [ Expr value.body ] |> fst |> List.distinct /// Extract all references from a type definition @@ -315,4 +338,18 @@ let extractFromType (typ : PT.PackageType.PackageType) : List = |> List.collect (fun case -> case.fields |> List.map (fun field -> TypeRef field.typ)) - extract roots |> List.distinct + extract roots |> fst |> List.distinct + + +/// The BUILTINS a function's body calls. Values and types cannot call one -- a type declaration has +/// no body, and a value's is evaluated through the same walk, so this is the fn entry point only +/// until that stops being true. +let builtinsInFn (fn : PT.PackageFn.PackageFn) : List = + snd (extract [ Expr fn.body ]) + + +/// The BUILTINS a value's body calls. A `val` body is an expression like any other. +let builtinsInValue + (value : PT.PackageValue.PackageValue) + : List = + snd (extract [ Expr value.body ]) diff --git a/backend/src/LibDB/PackageManager.fs b/backend/src/LibDB/PackageManager.fs index 9aade47bc0..1c5a49e9ba 100644 --- a/backend/src/LibDB/PackageManager.fs +++ b/backend/src/LibDB/PackageManager.fs @@ -663,6 +663,79 @@ let private sameSignature /// the only way back is an environment variable you have to know exists. With it, a rebind whose /// signature does not match what this build compiles against is refused, loudly, and the pinned /// version is used: a wrong edit degrades rather than bricks. +/// What the CURRENT BRANCH's overlay binds `Darklang..` to, if anything. +/// +/// `locations` is main's projection and has no branch column -- authoring two items on a branch +/// leaves zero rows there and two in `op_branches` -- so a store query cannot see a branch's work. +/// The overlay is already in memory as an op list, so this folds it directly: latest binding wins +/// per location, the same rule `createInMemoryOver` applies and the same one main's fold applies. +/// +/// Only TYPES go through this, and the asymmetry is deliberate. Resolving a FN ref through an +/// overlay would mean the binary runs a branch's parser or pretty-printer the moment you stand on +/// that branch, which is a real hazard and not one you opted into by switching. A type declaration +/// is inert: resolving it decides what a value is TAGGED with, and executes nothing. So a branch +/// may lend the kernel its types and may not lend it its code. +let private overlayTypeBinding + (modules : string list) + (name : string) + : string option = + let wanted : PT.PackageLocation = + { owner = "Darklang"; modules = modules; name = name } + + branchOverlayOps + |> List.fold + (fun acc op -> + match op with + | PT.PackageOp.SetName(loc, PT.PackageType(Hash h), _) when loc = wanted -> + Some h + | PT.PackageOp.Decision(_, + loc, + _, + PT.DecisionKind.Override(PT.PackageType(Hash h))) when + loc = wanted + -> + Some h + // A name bound to something that is not a type, or unbound outright, means the branch has + // taken this name away from the kernel. Fall back rather than keep an older binding. + | PT.PackageOp.SetName(loc, _, _) when loc = wanted -> None + | PT.PackageOp.Unbind(loc, _) when loc = wanted -> None + | _ -> acc) + None + + +/// Every Darklang-owned binding the CURRENT BRANCH's overlay adds or changes, as +/// (modules, name, itemType, hash). +/// +/// For `PackageRefsGenerator`, which builds the pinned hash file from `locations` and therefore has +/// the same main-only blind spot the resolver had. A branch that authors a type the kernel +/// references has to be able to produce a hash file naming it, or the F# on that git branch cannot +/// be built by anybody else. +let overlayDarklangBindings () : List = + let bindings = + System.Collections.Generic.Dictionary() + + for op in branchOverlayOps do + let apply loc target = + match target with + | PT.PackageType(Hash h) -> bindings[loc] <- ("type", h) + | PT.PackageValue(Hash h) -> bindings[loc] <- ("value", h) + | PT.PackageFn(Hash h) -> bindings[loc] <- ("fn", h) + + match op with + | PT.PackageOp.SetName(loc, target, _) -> apply loc target + | PT.PackageOp.Decision(_, loc, _, PT.DecisionKind.Override target) -> + apply loc target + | PT.PackageOp.Unbind(loc, _) -> bindings.Remove loc |> ignore + | _ -> () + + bindings + |> Seq.filter (fun kv -> kv.Key.owner = "Darklang") + |> Seq.map (fun kv -> + let (itemType, hash) = kv.Value + (String.concat "." kv.Key.modules, kv.Key.name, itemType, hash)) + |> List.ofSeq + + /// What `Darklang..` of this kind binds to on main, CACHED. /// /// Through `Caching.withCache`, which the fold clears, so this expires exactly when a rebinding @@ -782,8 +855,13 @@ let private resolveKernelTypeByName try let modulesStr = String.concat "." modules + // The branch first, then main. A branch that has authored this name is the thing you are + // standing on, and it is the whole reason F# on a git branch can reference package code + // authored on a dark branch. let candidateHash = - (kernelHashByName ("type", modulesStr, name) |> Ply.toTask).Result + match overlayTypeBinding modules name with + | Some h -> Some h + | None -> (kernelHashByName ("type", modulesStr, name) |> Ply.toTask).Result match candidateHash with | None -> None @@ -985,3 +1063,9 @@ let selectBranch (branchId : PT.BranchId) : unit = else branchOverlayOps <- (Branches.loadDeltaOps branchId).Result currentBranchIdOpt <- Some branchId + + // Moving branches changes what a NAME resolves to, and every cache here answers a question about + // a name. A one-shot `dark` selects its branch before resolving anything and never notices; a + // process that outlives a switch -- the LSP, the REPL, a daemon -- would otherwise answer for the + // branch it started on, forever. + Caching.invalidateAll () diff --git a/backend/src/LibDB/PackageOpPlayback.fs b/backend/src/LibDB/PackageOpPlayback.fs index 5ca304650b..092f91cf9f 100644 --- a/backend/src/LibDB/PackageOpPlayback.fs +++ b/backend/src/LibDB/PackageOpPlayback.fs @@ -33,6 +33,44 @@ open LibDB.PreparedBatch // Dependency table maintenance. // ------------------------------------------------------------------ +/// Record which BUILTINS an item's body calls. +/// +/// Separate from `updateDependencies` because a builtin edge is a different shape: not +/// content-addressed, so there is no hash to depend on and no location. This is the half of the +/// kernel/package-set interface that used to be invisible from the store, and the thing that lets +/// a fetched package set say which kernel it needs. +/// +/// ADDS, never replaces, for the same reason as the package edges: content is immutable, so what a +/// hash calls never changes. +let updateBuiltinDependencies + (ctx : Ctx) + (itemHash : string) + (builtins : List) + : Task = + task { + if List.isEmpty builtins then + () + else + let placeholders = + builtins + |> List.mapi (fun i _ -> $"($item_hash, $bname_{i}, $bver_{i})") + |> String.concat ", " + + let sql = + "INSERT OR IGNORE INTO package_builtin_deps " + + "(item_hash, builtin_name, builtin_version) VALUES " + + placeholders + + do! + exec ctx sql (fun cmd -> + p cmd "$item_hash" itemHash + builtins + |> List.iteri (fun i b -> + p cmd $"$bname_{i}" b.name + p cmd $"$bver_{i}" b.version)) + } + + /// Record what an item's body calls: one row per callee, by hash AND by the name this parse resolved /// it through. /// @@ -257,6 +295,7 @@ let private applyAddValue let refs = DE.extractFromValue value do! updateDependencies ctx hashStr refs + do! updateBuiltinDependencies ctx hashStr (DE.builtinsInValue value) } /// Apply a single AddFn op to the package_functions table. @@ -292,6 +331,7 @@ let private applyAddFn let refs = DE.extractFromFn fn do! updateDependencies ctx hashStr refs + do! updateBuiltinDependencies ctx hashStr (DE.builtinsInFn fn) } /// The `origin_ts` the log stamped on , or None when the log does not hold it. diff --git a/backend/src/LibDB/PackageRefsGenerator.fs b/backend/src/LibDB/PackageRefsGenerator.fs index 8423fd7f55..f2c4b383dc 100644 --- a/backend/src/LibDB/PackageRefsGenerator.fs +++ b/backend/src/LibDB/PackageRefsGenerator.fs @@ -91,7 +91,16 @@ let generate () : Ply = let hash = read.string "item_hash" (buildKey itemType modules name, hash)) - let dbMap = dbRows |> Map.ofList + // The branch's bindings go OVER main's. `locations` is main's projection, so without this a + // hash file generated while standing on a branch describes main and silently omits the very + // items the branch exists to add -- which is exactly the case where the file matters, because + // it is what lets somebody else build the F# that references them. + let dbMap = + PackageManager.overlayDarklangBindings () + |> List.fold + (fun acc (modules, name, itemType, hash) -> + Map.add (buildKey itemType modules name) hash acc) + (dbRows |> Map.ofList) // Preserves entries not found in the DB (e.g. RT types that share hashes with PT types and aren't in // locations), and, via `existingKeys` above, refs this process never registered. diff --git a/backend/src/LibDB/Purge.fs b/backend/src/LibDB/Purge.fs index 691365a58f..1d6dc2c1eb 100644 --- a/backend/src/LibDB/Purge.fs +++ b/backend/src/LibDB/Purge.fs @@ -32,6 +32,11 @@ let tables : List = "package_functions" "package_ops" "package_dependencies" + + // Which builtins each item calls. Derived from the bodies in the log by the same walk that + // fills `package_dependencies`, so a purged log has called nothing. + "package_builtin_deps" + "deprecations" // The LWW register for doc comments: what each `UpdateDoc` said and when. A claim about the diff --git a/backend/src/LibDB/Releases.fs b/backend/src/LibDB/Releases.fs index ca35f18d7a..959b074b25 100644 --- a/backend/src/LibDB/Releases.fs +++ b/backend/src/LibDB/Releases.fs @@ -245,6 +245,25 @@ let steps : List = print $" release: added `removed` to {List.length rows} stored conflict(s)" } + // What BUILTINS each item calls. New table, so `CREATE TABLE IF NOT EXISTS` in the schema + // would reach an existing store only because the bootstrap replays it, which it does not + // promise to. Named here so the store records having got it. Empty until the next fold + // rebuilds it, which is correct: it is derived. + { name = "20260912_000001_package_builtin_deps" + run = + fun () -> + Sql.query + "CREATE TABLE IF NOT EXISTS package_builtin_deps ( + item_hash TEXT NOT NULL, + builtin_name TEXT NOT NULL, + builtin_version INTEGER NOT NULL)" + |> Sql.executeStatementSync + + Sql.query + "CREATE UNIQUE INDEX IF NOT EXISTS idx_package_builtin_deps_unique + ON package_builtin_deps(item_hash, builtin_name, builtin_version)" + |> Sql.executeStatementSync } + // NEW STEPS GO ABOVE THIS LINE -- `scripts/migrations/new` appends here, and edits nothing else. ] diff --git a/backend/src/LibDB/Seed.fs b/backend/src/LibDB/Seed.fs index bcdc40eaa9..868bf9c7f7 100644 --- a/backend/src/LibDB/Seed.fs +++ b/backend/src/LibDB/Seed.fs @@ -96,6 +96,7 @@ let exportAt (outputPath : string) (upToCommit : string option) : Task = DELETE FROM package_values; DELETE FROM package_functions; DELETE FROM package_dependencies; + DELETE FROM package_builtin_deps; DELETE FROM deprecations; -- The builder's BRANCHES are not canon. A branch's ops live in `package_ops` at effective = 0 and are @@ -472,6 +473,7 @@ let projectionTables : List = "package_values" "locations" "package_dependencies" + "package_builtin_deps" "deprecations" // Folded from `Decision` ops; nothing else writes it. Being here is what makes it genuinely derived // rather than a second source of truth about the same decisions. diff --git a/backend/src/LibExecution/PackageRefs.fs b/backend/src/LibExecution/PackageRefs.fs index b6dfe81189..d6972e0c80 100644 --- a/backend/src/LibExecution/PackageRefs.fs +++ b/backend/src/LibExecution/PackageRefs.fs @@ -208,6 +208,35 @@ let private byNameEnabled : Lazy = let private currentStoreGeneration () : int = storeGeneration +/// Can this ref be resolved right now, from the store or the pin, WITHOUT raising? +/// +/// The ref closures are lazy, so an unresolvable ref is found whenever some code path happens to +/// reach it -- which can be much later than the build, in a command unrelated to whatever made it +/// unresolvable. This is the same lookup with the raise removed, so a caller can ask about every +/// ref at once and answer the real question: does this kernel agree with this package set? +let tryResolve + (kind : string) + (modules : string list) + (name : string) + : Option = + let fromStore = + if byNameEnabled.Force() then + match kind with + | "fn" -> resolveFnByName modules name + | "type" -> resolveTypeByName modules name + | _ -> None + else + None + + match fromStore with + | Some hash -> Some hash + | None -> + let fqn = $"""{kind}/{String.concat "." modules}.{name}""" + match getHashes () |> Map.tryFind fqn with + | Some hash when hash <> "" -> Some hash + | _ -> None + + /// Shared body of `Type.p` and `Fn.p`: a closure resolving `/.` /// against the hash file. Resolution is cached once per hash generation: the answer /// cannot change while the generation is stable, and resolving per call costs an @@ -276,11 +305,26 @@ let private makeRef if Map.isEmpty h then "" // Hash file not yet populated (CI before reload-packages) else - // A non-empty file missing this ref is stale: a ref was added, or an older - // binary regenerated it in place (`growIfNeeded` rewrites it). + // Neither the store nor the pin has it, and for a type there is nothing to degrade + // to -- a value has to be tagged with something. So this raises, and the message has + // to carry the whole situation, because the person reading it is usually mid-way + // through exactly the workflow this arc exists to support: F# that names package code + // which has not arrived yet. + let dotted = $"""Darklang.{String.concat "." modules}.{name}""" + + let message = + $"This build needs the {kind} `{dotted}`, and neither the package store nor " + + "`package-ref-hashes.txt` has it.\n" + + " Authoring it yourself: author it on your branch, then " + + "`scripts/run-local-exec refs generate`.\n" + + " Somebody else's, on a branch: `dark branch import ` then " + + "`dark switch `.\n" + + " Should be on main: your store is behind the kernel -- `dark pull`, or " + + "re-fetch the pinned package set." + Exception.raiseInternal - $"PackageRefs: {kind} hash not found. The hash file is stale; regenerate it with `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages`" - [ "fqn", fqn ] + message + [ "fqn", fqn; "kind", kind; "name", dotted ] module Type = @@ -710,3 +754,11 @@ let kernelHash () : string = |> sha.ComputeHash |> System.Convert.ToHexString |> fun s -> s.ToLowerInvariant().Substring(0, 16) + +/// Every ref the kernel declares, as (kind, modules, name). Populated at module init, so touching +/// this forces the whole table into existence. +let allRefs () : List = + let types = + Type._lookup |> Map.toList |> List.map (fun ((m, n), _) -> ("type", m, n)) + let fns = Fn._lookup |> Map.toList |> List.map (fun ((m, n), _) -> ("fn", m, n)) + types @ fns diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index 774ab194c5..06f2e0f0c0 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -123,6 +123,120 @@ module HandleCommand = return Error $"Export failed: {ex.Message}" } + /// Stand on the branch this rundir is on, the way the CLI does before it runs anything. + /// + /// Not global to LocalExec: the fill path deliberately refills MAIN from disk, and doing that + /// while standing on a branch would be wrong. Only the commands that ask a question ABOUT the + /// current branch select it. + let selectStoredBranch () : Ply = + uply { + match! LibDB.BranchSelection.select None None with + | Ok selection -> + LibDB.PackageManager.selectBranch ( + selection.branchId + |> Option.defaultValue LibExecution.ProgramTypes.BranchId.Main + ) + | Error _ -> () + } + + /// Does this kernel agree with the package set in front of it? + /// + /// Direction one of the two-way interface: every name the kernel references has to resolve, in + /// the store as seen from the current branch or in the pin. Direction two -- every builtin the + /// package set calls existing in this kernel -- needs the store to record builtin edges, which + /// it does not yet. + /// + /// Asked all at once, and at BUILD time, because the ref closures are lazy: an unresolvable ref + /// is otherwise found whenever some code path happens to reach it, which can be a different day + /// and an unrelated command. The case this exists for is checking out somebody's git branch + /// without their package work: the F# in your tree names things your store has never heard of, + /// and you should be told that then, in one list, rather than one at a time by whatever runs + /// first. + let checkRefs () : Ply> = + uply { + do! selectStoredBranch () + + let unresolved = + LibExecution.PackageRefs.allRefs () + |> List.filter (fun (kind, modules, name) -> + LibExecution.PackageRefs.tryResolve kind modules name |> Option.isNone) + + // Direction two: every builtin the package set calls has to exist in THIS kernel. Recorded + // by the fold in `package_builtin_deps`, which is what makes a store able to say which + // kernel it needs -- the check that used to answer this grepped `.dark` text off disk and + // stops being possible the day packages come from a seed. + let kernelBuiltins = + let b = Builtins.all () + Set.union + (b.fns.Values + |> Seq.map (fun f -> (f.name.name, f.name.version)) + |> Set.ofSeq) + (b.values.Values |> Seq.map (fun v -> (v.name.name, 0)) |> Set.ofSeq) + + let! calledBuiltins = + Sql.query + "SELECT DISTINCT builtin_name, builtin_version FROM package_builtin_deps" + |> Sql.executeAsync (fun read -> + (read.string "builtin_name", read.int "builtin_version")) + + let missingBuiltins = + calledBuiltins |> List.filter (fun b -> not (Set.contains b kernelBuiltins)) + + // An EMPTY table is not a pass. `package_builtin_deps` is a projection, so a store that got + // the table from a release step without re-folding has no rows, and the builtin half of the + // check would report success having asked nothing. Saying so is the difference between this + // check and one that quietly stops covering what it was written for. + if List.isEmpty calledBuiltins then + return + Error( + "this store records no builtin calls at all, so the builtin half of this check asked " + + "nothing. `package_builtin_deps` is a projection: re-fold the log to fill it " + + "(`scripts/build/reload-packages`, or any migration that drops projections)." + ) + elif List.isEmpty unresolved && List.isEmpty missingBuiltins then + let n = List.length (LibExecution.PackageRefs.allRefs ()) + print ( + $"All {n} kernel refs resolve, and all {List.length calledBuiltins} builtins this " + + "package set calls exist in this kernel." + ) + return Ok() + elif List.isEmpty unresolved then + let lines = + missingBuiltins + |> List.sort + |> List.map (fun (n, v) -> $" Builtin.{n} (v{v})") + |> String.concat "\n" + + return + Error( + $"this package set calls {List.length missingBuiltins} builtin(s) this kernel does " + + $"not have:\n{lines}\n\nA builtin was removed or renamed out from under package " + + "code that calls it. Land the package change that stops calling it, move the pin " + + "forward, and only then remove the builtin." + ) + else + let lines = + unresolved + |> List.sortBy (fun (kind, m, n) -> (kind, m, n)) + |> List.map (fun (kind, modules, name) -> + $""" {kind} Darklang.{String.concat "." modules}.{name}""") + |> String.concat + " +" + + return + Error( + $"{List.length unresolved} kernel ref(s) do not resolve against this package set: +" + + lines + + " + +This kernel and this package set do not agree. Usually that means the F# in " + + "your tree names package code your store does not have: import the branch bundle " + + "that goes with it, or move to the branch that has it." + ) + } + /// Write `package-ref-hashes.txt` from whatever store this rundir has. /// /// The kernel's entry points are pinned BY HASH, so a binary needs that file before it can resolve @@ -132,6 +246,7 @@ module HandleCommand = let generateRefs () : Ply> = uply { try + do! selectStoredBranch () do! LibDB.PackageRefsGenerator.generate () LibExecution.PackageRefs.reloadHashes () return Ok() @@ -213,6 +328,11 @@ let main (args : string[]) : int = $"Exporting seed at {commit} to {outputPath}" (HandleCommand.exportSeed outputPath (Some commit)) + | [ "refs"; "check" ] -> + handleCommand + "checking the kernel's refs against this package set" + (HandleCommand.checkRefs ()) + | [ "refs"; "generate" ] -> handleCommand "writing package-ref-hashes.txt from this store" @@ -241,6 +361,7 @@ let main (args : string[]) : int = print " migrations list" print " export-seed [commit]" print " refs generate" + print " refs check" print " pm-sweep-blobs" print " bench" print " bench-render" diff --git a/backend/tests/Tests/OpsProjections.Tests.fs b/backend/tests/Tests/OpsProjections.Tests.fs index 4dee582610..bdc58f376e 100644 --- a/backend/tests/Tests/OpsProjections.Tests.fs +++ b/backend/tests/Tests/OpsProjections.Tests.fs @@ -321,7 +321,7 @@ let durableReleaseCarriesForward = let registryCoversProjections = // The COUNT is in the name on purpose: adding a projection to the registry without adding it here // is exactly the drift this catches. - test "the projection registry covers exactly the 7 regenerable projections" { + test "the projection registry covers exactly the 8 regenerable projections" { Expect.equal (List.sort Seed.projectionTables) (List.sort @@ -330,6 +330,7 @@ let registryCoversProjections = "package_values" "locations" "package_dependencies" + "package_builtin_deps" "deprecations" "propagation_policy" ]) "the registry's tables are exactly Seed.export's stripped projections (incl. deprecations)" diff --git a/scripts/build/_buildplan.py b/scripts/build/_buildplan.py index be27d8f759..c832f67e29 100644 --- a/scripts/build/_buildplan.py +++ b/scripts/build/_buildplan.py @@ -38,6 +38,7 @@ class Should: "backend_quick_build", "run_migrations", "reload_all_packages", + "check_refs", "backend_test", "circleci_validate", "shellcheck", @@ -52,6 +53,7 @@ def __init__(self): self.backend_full_build = False self.backend_test = False self.reload_all_packages = False + self.check_refs = False self.circleci_validate = False self.run_migrations = False self.shellcheck = [] @@ -183,6 +185,12 @@ def expand(should, run_tests=False): if s.run_migrations and not packages_from_seed(): s.reload_all_packages = True + # Whenever either side of the kernel/package-set interface could have moved. An `.fs` + # change can add or move a ref; a package change can take away what one resolves to. + # Cheap enough to run on both rather than reason about which refs a change touched. + if s.backend_quick_build or s.backend_full_build or s.reload_all_packages: + s.check_refs = True + # backend_test is set by execute() on any build, but run_test() no-ops without # --test, so a plan that lists it would be lying. if not run_tests: diff --git a/scripts/build/compile b/scripts/build/compile index 02c89be790..2ae19750ff 100755 --- a/scripts/build/compile +++ b/scripts/build/compile @@ -211,6 +211,17 @@ def run_migrations(): return run_backend(start, f"scripts/build/run-migrations {configuration}") +def check_refs(): + """Does the kernel this build produced agree with the package set in front of it? + + The ref closures are lazy, so an unresolvable ref is otherwise found whenever some code + path happens to reach it, which can be a different day and an unrelated command. The + case this exists for is checking out somebody's git branch without their package work. + """ + start = time.time() + return run_backend(start, "scripts/run-local-exec refs check") + + def reload_all_packages(): start = time.time() if optimize: @@ -299,6 +310,14 @@ def execute(should, outcome): if should.reload_all_packages: step("reload_all_packages", reload_all_packages) + # After anything that could have moved either side of the kernel/package-set interface. + # Decided here rather than in the plan because execute() sets some of these itself, so the + # plan's answer is not the final one. Cheap: one pass over ~200 refs against caches the + # store already holds. + if should.check_refs or should.reload_all_packages or should.backend_quick_build \ + or should.backend_full_build: + step("check_refs", check_refs) + # Gated here rather than inside backend_test(), so a build that skipped the tests # doesn't list running them among the things it did. if should.backend_test and run_tests: diff --git a/scripts/testing/test-build-planning.py b/scripts/testing/test-build-planning.py index 41d8b0252e..56519c3e3e 100755 --- a/scripts/testing/test-build-planning.py +++ b/scripts/testing/test-build-planning.py @@ -153,19 +153,22 @@ def test_fsharp_change_reaches_the_package_reload(self): with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="disk"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), - ["backend_quick_build", "run_migrations", "reload_all_packages"]) + ["backend_quick_build", "run_migrations", "reload_all_packages", "check_refs"]) with Env(CI="true", DARK_CONFIG_PACKAGES_SOURCE="disk"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), - ["backend_full_build", "run_migrations", "reload_all_packages"]) + ["backend_full_build", "run_migrations", "reload_all_packages", "check_refs"]) def test_fsharp_change_stops_at_migrations_from_seed(self): # Half the cost of an F# change today is a package reload that usually did not need to # happen. In seed mode it does not happen at all. + # + # `check_refs` still does, and has to: an F# change can add or move a kernel ref, and in + # seed mode there is no reload to notice that the store cannot answer for it. with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="seed"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), - ["backend_quick_build", "run_migrations"]) + ["backend_quick_build", "run_migrations", "check_refs"]) def test_full_build_replaces_the_quick_one(self): actions = self.expand("backend/src/LibDB/LibDB.fsproj") From 97cc636ff4a6ca46accbac450870d1765dd5e421 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:26:22 -0400 Subject: [PATCH 20/32] Cache a pinned seed per machine, not per clone Today `git clone && scripts/dev/start` works from the repo alone. After the flip it needs the network and a live server, which is a robustness regression I had not considered at all and which matters most for exactly the person with the least: a contributor on a plane, or anyone when the server is down. A seed cut at a COMMIT never changes, so it is worth keeping once per machine. With this, a machine needs the network once per pin; a second clone, a re-clone, or working offline is a file copy. Only for a PINNED fetch -- `latest` is not immutable and must not be cached under a name that claims it is. And when there is nothing to fall back to, the failure names the way out (`--from `) rather than being a bare curl error. Tested three ways: cold with the server up, a second clone with the server killed, and a third with the cache deleted too. --- scripts/fetch-seed | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/scripts/fetch-seed b/scripts/fetch-seed index 834b9df8c3..dcdccea233 100755 --- a/scripts/fetch-seed +++ b/scripts/fetch-seed @@ -87,8 +87,44 @@ else SEED_URL="$DEFAULT_URL" fi - echo "Fetching seed from $SEED_URL" - curl -fL -o "$DEST_SEED" "$SEED_URL" + # A seed cut at a COMMIT never changes, so it is worth keeping once per machine rather than once + # per clone. Without this, every clone and every re-clone needs the network -- which is a real + # robustness regression against today, where `git clone && scripts/dev/start` works from the repo + # alone. With it, a machine needs the network once per pin, and a second clone, a re-clone, or + # working on a plane is a file copy. + # + # Only for a pinned fetch. `latest` is not immutable and must not be cached under a name that + # claims it is. + CACHE_DIR="${DARK_SEED_CACHE:-$HOME/.darklang/seeds}" + CACHED="" + if [[ -n "$COMMIT" ]]; then + CACHED="${CACHE_DIR}/${COMMIT}.db" + fi + + if [[ -n "$CACHED" && -f "$CACHED" ]]; then + echo "Using the cached seed for ${COMMIT} (${CACHED})" + cp "$CACHED" "$DEST_SEED" + else + echo "Fetching seed from $SEED_URL" + if ! curl -fL -o "$DEST_SEED" "$SEED_URL"; then + { + echo + echo "Could not fetch a seed from ${SEED_URL}." + if [[ -n "$COMMIT" ]]; then + echo "Nothing on this machine has that commit cached, either." + fi + echo "If you have a seed file already, install it directly and skip the network:" + echo " scripts/fetch-seed --from " + } >&2 + exit 1 + fi + + if [[ -n "$CACHED" ]]; then + mkdir -p "$CACHE_DIR" + cp "$DEST_SEED" "$CACHED" + echo "Cached for this machine at ${CACHED}" + fi + fi fi SEED_SIZE=$(du -h "$DEST_SEED" | cut -f1) From 5206ff8f58fcb6919af4460e1f7f5afd465aa56f Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:42:52 -0400 Subject: [PATCH 21/32] Check that a resolved ref names content the store actually has `tryResolve` accepted any non-empty hash, so a pin naming content the store no longer holds counted as resolving. That is the failure measured this morning: a stale type hash produces no error and blank output at runtime. The check now verifies the hash against the set of hashes the store holds content for, asked once rather than per ref, so it is caught at build time instead. And when the check fails and git is on a branch with a same-named dark branch, say the command: git is on `add-foo` and there is a dark branch called `add-foo`, but you are on dark main. Try `dark switch add-foo`. That is the F#/package coupling made visible at the one moment it matters, rather than a rule somebody has to have read. Deliberately NOT an auto-switching "dark branch follows git branch" tier. Every ordering I worked through either overrides an explicit `dark switch` or fails to let one stick, and it is not a thing I can test on a person. A hint that carries the fix is the version worth shipping without that. Finding the git branch walks UP for `.git` rather than assuming the rundir sits directly inside the repo: true for the dev rundir, false for a test's, and silent either way. --- backend/src/LocalExec/LocalExec.fs | 85 +++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index 06f2e0f0c0..cb7b30af9b 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -139,6 +139,42 @@ module HandleCommand = | Error _ -> () } + /// The git branch this tree is checked out on, if it is a git tree at all. + /// + /// Read out of `.git/HEAD` rather than by shelling out: this runs inside the build, and a + /// process spawn for one line of text is not worth it. A detached HEAD answers `None`, which is + /// right -- there is no branch NAME to line up with. + let private gitBranchName () : Option = + try + // Walk UP looking for `.git`, rather than assuming the rundir sits directly inside the + // repo. It does for the dev rundir and does not for a test's, and the difference is silent: + // you get no branch name and no error. + let rec findGitHead (dir : System.IO.DirectoryInfo) : string option = + if isNull (box dir) then + None + else + let candidate = System.IO.Path.Combine(dir.FullName, ".git", "HEAD") + if System.IO.File.Exists candidate then + Some candidate + else + findGitHead dir.Parent + + let head = + match findGitHead (System.IO.DirectoryInfo LibConfig.Config.runDir) with + | Some h -> h + | None -> "" + + if head <> "" && System.IO.File.Exists head then + let text = (System.IO.File.ReadAllText head).Trim() + let prefix = "ref: refs/heads/" + + if text.StartsWith prefix then Some(text.Substring prefix.Length) else None + else + None + with _ -> + None + + /// Does this kernel agree with the package set in front of it? /// /// Direction one of the two-way interface: every name the kernel references has to resolve, in @@ -156,10 +192,26 @@ module HandleCommand = uply { do! selectStoredBranch () + // Every hash this store actually HOLDS content for. A ref resolving to a hash is not the + // same as that hash naming anything: a pin can name content the store no longer has, and + // that is precisely the failure this check exists to catch -- it does not error at runtime, + // it renders blank. Asked once as a set rather than per ref. + let! knownTypes = + Sql.query "SELECT hash FROM package_types" + |> Sql.executeAsync (fun read -> read.string "hash") + + let! knownFns = + Sql.query "SELECT hash FROM package_functions" + |> Sql.executeAsync (fun read -> read.string "hash") + + let known = Set.union (Set.ofList knownTypes) (Set.ofList knownFns) + let unresolved = LibExecution.PackageRefs.allRefs () |> List.filter (fun (kind, modules, name) -> - LibExecution.PackageRefs.tryResolve kind modules name |> Option.isNone) + match LibExecution.PackageRefs.tryResolve kind modules name with + | None -> true + | Some hash -> not (Set.contains hash known)) // Direction two: every builtin the package set calls has to exist in THIS kernel. Recorded // by the fold in `package_builtin_deps`, which is what makes a store able to say which @@ -224,16 +276,33 @@ module HandleCommand = " " + let! hint = + uply { + // If git is on a branch and a dark branch of the same name exists, that is almost + // certainly where the missing items are -- so say the command rather than the + // category. The coupling made visible at the one moment it matters, instead of a + // rule somebody has to have read. + match gitBranchName () with + | None -> return "" + | Some git -> + let! darkBranch = LibDB.Branches.liveIdForName git + + match darkBranch with + | Some _ when LibDB.PackageManager.currentBranchId () = BranchId.Main -> + return + $"\n\ngit is on `{git}` and there is a dark branch called `{git}`, " + + $"but you are on dark main. Try `dark switch {git}`." + | _ -> return "" + } + return Error( - $"{List.length unresolved} kernel ref(s) do not resolve against this package set: -" + $"{List.length unresolved} kernel ref(s) do not resolve against this package set:\n" + lines - + " - -This kernel and this package set do not agree. Usually that means the F# in " - + "your tree names package code your store does not have: import the branch bundle " - + "that goes with it, or move to the branch that has it." + + $"\n\nThis kernel and this package set do not agree. Usually that means the " + + "F# in your tree names package code your store does not have: import the branch " + + "bundle that goes with it, or move to the branch that has it." + + hint ) } From b54df1c755df9de0dae227d1548d9acfbc754870 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:46:32 -0400 Subject: [PATCH 22/32] Bound the server's seed cache `/seed/.db` is anonymous -- it has to be, since fetching a pinned package set is the one thing a contributor with no access must be able to do -- and it kept a ~12MB file per commit asked for. The commits are discoverable through the equally public `/sync/pull`, so a stranger could walk a store's history and make the server keep a file for every commit in it. Eight cuts, oldest-by-mtime evicted, which is least-recently-asked-for because a cut is only written when it is missing. Housekeeping failures are ignored: refusing to serve a seed because a stale file would not delete is the wrong trade. The first version raised inside the handler and turned every seed request into a 500, because `8L` is an Int64 while `List.length`, `List.drop` and `getModifiedTime` are all `Int`. The probe counted files on disk and never looked at the status code, so it reported the cuts as present while the endpoint was erroring. It asserts the codes now. --- packages/darklang/sync/relay/protocol.dark | 54 +++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/darklang/sync/relay/protocol.dark b/packages/darklang/sync/relay/protocol.dark index 4f96c0e2ba..4e7f3752c1 100644 --- a/packages/darklang/sync/relay/protocol.dark +++ b/packages/darklang/sync/relay/protocol.dark @@ -291,6 +291,56 @@ let headCommit () : String = | [] -> "" +/// How many cut seeds this server keeps. At roughly twelve megabytes each, eight is about a +/// hundred megabytes, and more than anybody browsing pins needs at once. +val seedCacheKeep = 8 + + +/// Drop the oldest cuts past `seedCacheKeep`. +/// +/// Not tidiness. `/seed/.db` is ANONYMOUS -- it has to be, because fetching a pinned +/// package set is the one thing a contributor with no access must be able to do -- and it keeps a +/// file per commit asked for. The commits are discoverable through the equally public +/// `/sync/pull`, so without a bound a stranger can walk a store's history and make the server keep +/// a twelve-megabyte file for every commit in it. +/// +/// Oldest by modified time, which is "least recently asked for" because a cut is rewritten only +/// when it is missing. Failures are ignored: this is housekeeping, and refusing to serve a seed +/// because a stale file would not delete would be the wrong trade. +let pruneSeedCache () : Unit = + let dir = Stdlib.Cli.Path.parent (seedCachePath "x") + + match Stdlib.Cli.Dir.list dir with + | Error _ -> () + | Ok entries -> + let cuts = + entries + |> Stdlib.List.filter (fun e -> Stdlib.String.endsWith e ".db") + |> Stdlib.List.filterMap (fun e -> + let full = Stdlib.Cli.Path.join [ dir, e ] + + match Stdlib.Cli.File.getModifiedTime full with + | Ok t -> Stdlib.Option.Option.Some((t, full)) + | Error _ -> Stdlib.Option.Option.None) + + let total = Stdlib.List.length cuts + + if total <= seedCacheKeep then + () + else + // Oldest first, then delete exactly the excess. Ascending rather than descending because + // every count here is an `Int` and negating one to reverse the sort is how this got an + // Int/Int64 mismatch the first time, which raised inside the handler and turned every seed + // request into a 500. + cuts + |> Stdlib.List.sortBy (fun c -> let (t, _) = c in t) + |> Stdlib.List.take (total - seedCacheKeep) + |> Stdlib.List.iter (fun c -> + let (_, path) = c + let _ = Stdlib.Cli.File.delete path + ()) + + /// Cut a seed at , or hand back the one already cut there. let cutSeedAt (commit: String) : Stdlib.Result.Result = let path = seedCachePath commit @@ -305,7 +355,9 @@ let cutSeedAt (commit: String) : Stdlib.Result.Result = | Ok _ -> match Stdlib.LocalStore.seedTo path (Stdlib.Option.Option.Some commit) with | Error e -> Stdlib.Result.Result.Error e - | Ok _ -> Stdlib.Result.Result.Ok path + | Ok _ -> + let _ = pruneSeedCache () + Stdlib.Result.Result.Ok path /// What a seed cut from this relay says about itself, so a pin can be checked without downloading From 386a0130f87b1d6aa639581b4fe3669cd7a0e7bd Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:48:25 -0400 Subject: [PATCH 23/32] Carry a dark branch's work in the git branch that needs it An F# change can reference package code, and then the two have to travel, merge and be abandoned together. Git already does that for the F#; this lets it do the same for the package half, so one PR carries both and reviewing or dropping it is one act. scripts/packages/bundle export|import|show Export is EXPLICIT, like `git add`: it writes into your tree and that should be something you asked for. Forgetting is caught rather than silent -- `refs check` fails the build for anyone whose F# names package code they do not have, and names the branch to switch to when there is one. Import is AUTOMATIC, from `prepare-package-set`, because checking out the git branch IS asking for that branch's package code. Idempotent: ops are content-addressed, so the second import says "up to date". Going back to dark main REMOVES the bundle rather than leaving it. A bundle that outlives its branch is how somebody imports work that was merged weeks ago. Measured: a type plus a function is about a kilobyte of JSON, so the bundle is smaller than the `package-ref-hashes.txt` diff it travels with, and `bundle show` gives a reviewer the branch and the op count without fetching anything. Stachu's shape was "the PR names a dark branch and the merge button coordinates"; this is the other half of that, not an alternative to it. The NAME is what merge coordination acts on; the BUNDLE is what lets somebody with no server access participate at all. --- scripts/build/prepare-package-set | 16 ++++++- scripts/packages/bundle | 73 +++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100755 scripts/packages/bundle diff --git a/scripts/build/prepare-package-set b/scripts/build/prepare-package-set index 15ce6beac4..a3a349a56c 100755 --- a/scripts/build/prepare-package-set +++ b/scripts/build/prepare-package-set @@ -26,11 +26,23 @@ while [[ $# -gt 0 ]]; do esac done +# The branch bundle, if this git branch carries one. Imported whichever way the base package set +# arrives, because checking out the git branch IS asking for that branch's package code, and the +# F# in the tree may name items only the bundle has. Idempotent: ops are content-addressed, so +# re-importing what is already here changes nothing. +import_bundle() { + if [[ -f "${DARK_PACKAGE_BUNDLE:-package-branch.json}" ]]; then + scripts/packages/bundle import + fi +} + PIN=$(awk '$1 == "commit" { print $2 }' package-set.txt) if [[ -z "$PIN" || "$PIN" == "unset" ]]; then echo "No pin in package-set.txt; building the package set from packages/." - exec scripts/build/reload-packages + scripts/build/reload-packages + import_bundle + exit 0 fi echo "Pinned to ${PIN}; fetching the seed rather than reloading packages/." @@ -40,3 +52,5 @@ scripts/fetch-seed "${URL_ARGS[@]}" --force # projection of it. Without this step every kernel lookup resolves to "" and the binary fails on # its first command, somewhere that says nothing about why. scripts/run-local-exec refs generate + +import_bundle diff --git a/scripts/packages/bundle b/scripts/packages/bundle new file mode 100755 index 0000000000..d4ecd1d09b --- /dev/null +++ b/scripts/packages/bundle @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# The package work on your dark branch, as a file that travels with your git branch. +# +# scripts/packages/bundle export write your dark branch to package-branch.json +# scripts/packages/bundle import recreate that branch from the file +# scripts/packages/bundle show what is in the file +# +# Why a file in the repo. An F# change can reference package code, and then the two have to +# travel, merge and be abandoned together. Git already does that for the F#; this lets it do the +# same for the package half, so one PR carries both and reviewing or dropping it is one act. +# +# EXPORT is explicit, like `git add`: producing it writes into your tree and that should be +# something you asked for. Forgetting is caught rather than silent -- `refs check` fails the build +# for anyone whose F# names package code they do not have. +# +# IMPORT is automatic (`scripts/build/prepare-package-set` runs it), because checking out the git +# branch IS asking for that branch's package code. Importing is idempotent: ops are +# content-addressed, so re-importing what you already have changes nothing. + +set -euo pipefail + +[[ -f package-set.txt ]] || { echo "bundle: run from the repo root" >&2; exit 2; } + +BUNDLE="${DARK_PACKAGE_BUNDLE:-package-branch.json}" +CLI="${CLI:-./scripts/run-cli}" + +usage() { sed -n '3,19p' "$0" | sed 's/^# \?//'; } + +case "${1:-}" in + export) + branch=$("$CLI" branch 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | tail -1) + if [[ "$branch" == *"on main"* || -z "$branch" ]]; then + # Nothing to carry. Remove a stale bundle rather than leave one describing work that is + # now on main: a bundle outliving its branch is how somebody imports a branch that was + # merged weeks ago. + if [[ -f "$BUNDLE" ]]; then + rm -f "$BUNDLE" + echo "On dark main; removed $BUNDLE." + else + echo "On dark main; nothing to bundle." + fi + exit 0 + fi + + name=$(sed -E 's/^on branch "([^"]+)".*/\1/' <<<"$branch") + "$CLI" branch export "$name" "$BUNDLE" > /dev/null + echo "Wrote $BUNDLE from dark branch \"$name\" ($(wc -c < "$BUNDLE") bytes)." + echo "Commit it alongside the F# that needs it." + ;; + + import) + if [[ ! -f "$BUNDLE" ]]; then + echo "No $BUNDLE; nothing to import." + exit 0 + fi + "$CLI" branch import "$BUNDLE" + ;; + + show) + [[ -f "$BUNDLE" ]] || { echo "No $BUNDLE." >&2; exit 1; } + python3 -c " +import json,sys +b=json.load(open('$BUNDLE')) +print('branch:', b.get('name'), '(' + str(b.get('branchId','?'))[:8] + ')') +print('ops: ', len(b.get('ops', []))) +print('commits:', len(b.get('commits', []))) +" + ;; + + -h|--help|"") usage ;; + *) echo "bundle: unknown argument $1" >&2; usage >&2; exit 2 ;; +esac From b5f282a43bdeeca19b2c700c113df7abaf23cf6d Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 16:51:29 -0400 Subject: [PATCH 24/32] Never let caching a seed fail the fetch The seed is fetched and installed before the cache is written, so a cache that cannot be written costs the next fetch a download and nothing else. Failing there instead turns an unwritable home directory into a failed build, which is what it did: the `seed-serving` gate downloaded 12.3MB successfully and then died on `mkdir: /home/dark/.darklang/seeds: Permission denied`. Same lesson as the server's cache prune, paid twice in one afternoon: housekeeping must not be able to fail the thing it is housekeeping for. Reads are best-effort too -- an unreadable cached file falls through to the network rather than erroring. --- scripts/fetch-seed | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/fetch-seed b/scripts/fetch-seed index dcdccea233..0546e655e5 100755 --- a/scripts/fetch-seed +++ b/scripts/fetch-seed @@ -101,9 +101,8 @@ else CACHED="${CACHE_DIR}/${COMMIT}.db" fi - if [[ -n "$CACHED" && -f "$CACHED" ]]; then + if [[ -n "$CACHED" && -r "$CACHED" ]] && cp "$CACHED" "$DEST_SEED" 2>/dev/null; then echo "Using the cached seed for ${COMMIT} (${CACHED})" - cp "$CACHED" "$DEST_SEED" else echo "Fetching seed from $SEED_URL" if ! curl -fL -o "$DEST_SEED" "$SEED_URL"; then @@ -119,10 +118,16 @@ else exit 1 fi + # Best-effort, always. The seed is already fetched and installed by this point, so a cache + # that cannot be written costs the NEXT fetch a download and nothing else. Failing here + # instead would turn an unwritable home directory into a failed build, which is what happened + # the first time this shipped. if [[ -n "$CACHED" ]]; then - mkdir -p "$CACHE_DIR" - cp "$DEST_SEED" "$CACHED" - echo "Cached for this machine at ${CACHED}" + if mkdir -p "$CACHE_DIR" 2>/dev/null && cp "$DEST_SEED" "$CACHED" 2>/dev/null; then + echo "Cached for this machine at ${CACHED}" + else + echo "(could not cache to ${CACHE_DIR}; the seed is installed regardless)" + fi fi fi fi From 7f614383cd452f66dd858b7f6946bd121e44bdb1 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 17:12:17 -0400 Subject: [PATCH 25/32] Let a reviewer read a bundle without importing it, and warn on pin drift `bundle show --source` renders what the bundle changes, with the full declarations, by importing into a DISPOSABLE store. Reviewing a stranger's branch should not mean taking their ops into the store you work in, and a PR reviewer is exactly the person with the least reason to trust the bundle in front of them. "review-me" changes 2 names vs its parent: Darklang.LanguageTools + Reviewed [type] dd5ab310 type Reviewed = { a: Int64 b: String } + useReviewed [fn] 4ac94fb1 let useReviewed (): Int64 = 7L And `bundle export` warns when your main has moved past the pin. A bundle carries the BRANCH's ops; if your branch rests on commits only you have, the bundle is not enough for anyone else, who holds the pinned set plus this file and nothing more. CI says so too, as a failed `refs check`, but by then it is somebody else's red build. Asking `--branch main` explicitly, because `dark log` shows a BRANCH's ops when you are standing on one -- the bare form reported "your main is at AddFn". --- scripts/packages/bundle | 43 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/scripts/packages/bundle b/scripts/packages/bundle index d4ecd1d09b..e854403603 100755 --- a/scripts/packages/bundle +++ b/scripts/packages/bundle @@ -5,6 +5,7 @@ # scripts/packages/bundle export write your dark branch to package-branch.json # scripts/packages/bundle import recreate that branch from the file # scripts/packages/bundle show what is in the file +# scripts/packages/bundle show --source ...and what it actually changes # # Why a file in the repo. An F# change can reference package code, and then the two have to # travel, merge and be abandoned together. Git already does that for the F#; this lets it do the @@ -20,7 +21,8 @@ set -euo pipefail -[[ -f package-set.txt ]] || { echo "bundle: run from the repo root" >&2; exit 2; } +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" +[[ -f "$PINFILE" ]] || { echo "bundle: no $PINFILE (run from the repo root?)" >&2; exit 2; } BUNDLE="${DARK_PACKAGE_BUNDLE:-package-branch.json}" CLI="${CLI:-./scripts/run-cli}" @@ -47,6 +49,24 @@ case "${1:-}" in "$CLI" branch export "$name" "$BUNDLE" > /dev/null echo "Wrote $BUNDLE from dark branch \"$name\" ($(wc -c < "$BUNDLE") bytes)." echo "Commit it alongside the F# that needs it." + + # A bundle carries the BRANCH's ops. If your main has moved past the pin -- because you + # pulled -- your branch may rest on commits nobody else has, and the bundle will not be + # enough for them: they hold the pinned set plus this file and nothing else. CI says so + # too, as a failed `refs check`, but by then it is somebody else's red build. + pin=$(awk '$1 == "commit" { print $2 }' "$PINFILE") + if [[ -n "$pin" && "$pin" != "unset" ]]; then + # `--branch main` explicitly: `dark log` shows a BRANCH's ops when you are standing on + # one, and we are, so the bare form answers with an op kind rather than a commit hash. + head=$("$CLI" --branch main log 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' \ + | awk 'NR>3 { print $1; exit }') + if [[ -n "$head" && "${pin:0:${#head}}" != "$head" ]]; then + echo + echo "Note: your main is at ${head}, but ${PINFILE} pins ${pin:0:8}." + echo "If this branch rests on commits the pin does not have, this bundle will not be" + echo "enough for anyone else. Bump the pin, or wait for it to catch up." + fi + fi ;; import) @@ -60,12 +80,31 @@ case "${1:-}" in show) [[ -f "$BUNDLE" ]] || { echo "No $BUNDLE." >&2; exit 1; } python3 -c " -import json,sys +import json b=json.load(open('$BUNDLE')) print('branch:', b.get('name'), '(' + str(b.get('branchId','?'))[:8] + ')') print('ops: ', len(b.get('ops', []))) print('commits:', len(b.get('commits', []))) " + if [[ "${2:-}" == "--source" ]]; then + # Rendered by importing into a DISPOSABLE store, never yours. Reviewing a stranger's + # branch should not mean taking their ops into the store you work in, and a PR reviewer + # is exactly the person with the least reason to trust the bundle in front of them. + scratch=$(mktemp -d) + # shellcheck disable=SC2064 # expand now: $scratch must not be re-evaluated at trap time + trap "rm -rf '$scratch'" EXIT + scripts/testing/_copy-store "$scratch/data.db" > /dev/null + name=$(python3 -c " +import json +print(json.load(open('$BUNDLE')).get('name','')) +") + echo + env DARK_CONFIG_RUNDIR="$scratch" HOME="$scratch" \ + ./backend/Build/out/Cli/Debug/net10.0/Cli branch import "$BUNDLE" > /dev/null 2>&1 + env DARK_CONFIG_RUNDIR="$scratch" HOME="$scratch" \ + ./backend/Build/out/Cli/Debug/net10.0/Cli branch diff "$name" --source 2>&1 \ + | sed 's/\x1b\[[0-9;]*m//g' + fi ;; -h|--help|"") usage ;; From 1c6f92368bbcdf7d4ee65f2e8f21297bad44e261 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 17:34:51 -0400 Subject: [PATCH 26/32] dark edit : the whole module at once The replacement for opening a `.dark` file, and after the flip there is no file to open. It renders every declaration directly under the module, hands you $EDITOR, and applies the result through `dark module` -- which validates all of them before saving any, so a half-finished edit lands nothing rather than half. The design question was deletion by omission, and it is where this command would lose work. `dark module` adds and updates; it does not unbind. So leaving a declaration out of the file could have meant "remove it" -- and one mis-parse that dropped an item from the RENDER would then quietly end a name. It does not guess: omissions are named back to you and left alone, and `dark remove` stays the way a name ends, in one op, confirmed, visible in `dark status`. The report only fires after an apply that landed. A parse error lands nothing at all, and saying "left alone" about it reads as though the rest went in. Render order is types, then values, then fns, each alphabetical -- deterministic, or a render-edit-save round trip churns the file for nobody. Everything else is the single-item path's, which already got the risky parts right: a non-zero editor exit applies nothing and keeps your file, and a rejection prints where your edit still is. A whole module is more to lose. One existing assertion said "a module is refused"; it now checks that a module with no terminal points at the file form. That test caught the behaviour change the day it happened, which is what it is for. --- backend/tests/Tests/CliPackages.Tests.fs | 79 +++++++- backend/tests/Tests/CliScm.Tests.fs | 8 +- packages/darklang/cli/packages/edit.dark | 219 ++++++++++++++++++++++- 3 files changed, 295 insertions(+), 11 deletions(-) diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index fc74ab9b1d..037077c8f4 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -1059,8 +1059,85 @@ let renameIsVisibleToEverythingThatReads = }) +/// `dark edit ` is the replacement for opening a `.dark` file, so the properties that +/// matter are the ones a file gave you for free: everything applies together, and nothing you did +/// not ask to remove goes away. +/// +/// Driven through the FILE form, which needs no terminal. The editor form is the same code past +/// the point where it has a file. +let editModuleAppliesTogetherAndRemovesNothing = + instanceTest + "editing a module applies every declaration at once, and omitting one leaves it alone" + (fun state -> + task { + do! start state + + let write (contents : string) : string = + let path = System.IO.Path.GetTempFileName() + ".dark" + System.IO.File.WriteAllText(path, contents) + path + + do! + run + state + [ "module" + "/Tests.EditMod" + write "let one (): Int64 = 1L\nlet two (): Int64 = 2L\n" ] + + // One changed, one untouched, applied as a batch. + do! + run + state + [ "edit" + "Tests.EditMod" + write "let one (): Int64 = 111L\nlet two (): Int64 = 2L\n" ] + + do! + shows + state + [ "eval"; "Tests.EditMod.one ()" ] + "111" + "the edited one landed" + do! + shows + state + [ "eval"; "Tests.EditMod.two ()" ] + "2" + "its sibling is untouched" + + // Leaving a declaration OUT must not end it. Guessing the other way is where this command + // would lose work: one mis-parse that dropped an item from the render would unbind a name. + do! + shows + state + [ "edit"; "Tests.EditMod"; write "let one (): Int64 = 222L\n" ] + "left alone, not in your file" + "the omitted declaration is named back" + + do! shows state [ "eval"; "Tests.EditMod.two ()" ] "2" "and is still bound" + + // A declaration that does not parse lands NOTHING, not the half that did. + do! + run + state + [ "edit" + "Tests.EditMod" + write "let one (): Int64 = 999L\nlet bad (): Int64 = @@@\n" ] + + do! + shows + state + [ "eval"; "Tests.EditMod.one ()" ] + "222" + "a batch with one bad declaration applies none of it" + + do! discardAll state + }) + + let tests : List = - [ lsNamesWhatIsThere + [ editModuleAppliesTogetherAndRemovesNothing + lsNamesWhatIsThere treeShowsDescendants viewPrintsSource viewRefusesWhatIsNotThere diff --git a/backend/tests/Tests/CliScm.Tests.fs b/backend/tests/Tests/CliScm.Tests.fs index 54246982f6..b6596877fb 100644 --- a/backend/tests/Tests/CliScm.Tests.fs +++ b/backend/tests/Tests/CliScm.Tests.fs @@ -1658,12 +1658,16 @@ let private editChangesAnItemWithoutRetypingIt = "usage: dark edit" "a bare `edit` says how to use it" + // A module is EDITABLE now (`Packages.Edit.editModule`), so what this asserts is the + // shape without a terminal: it points at the file form rather than spawning an editor + // into a pipe. The old expectation here was "a module is refused", which is how this + // test earned its place -- it caught the behaviour change the day it happened. do! shows state [ "edit"; "Tests.Edit" ] - "is a module" - "a module is refused, and named as the reason" + "applies a file instead" + "a module with no terminal points at the file form" do! shows diff --git a/packages/darklang/cli/packages/edit.dark b/packages/darklang/cli/packages/edit.dark index 73b24d91c3..b0d77e14a0 100644 --- a/packages/darklang/cli/packages/edit.dark +++ b/packages/darklang/cli/packages/edit.dark @@ -78,15 +78,13 @@ let execute (state: AppState) (args: List) : AppState = match itemLocation location with | None -> - Stdlib.printLine - (Stdlib.Cli.UI.Colors.error - $"{name} is a module. `dark edit` takes one fn, type or value.") - - Stdlib.printLine - (Stdlib.Cli.UI.Colors.dimText - " `dark module /Some.Module ` edits a whole module at once.") + match location with + | Module modulePath -> editModule state modulePath fileArg + | _ -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {name}.") - state + state | Some loc -> match fileArg with @@ -203,6 +201,13 @@ let help (_state: AppState) : String = [ "" " dark edit open $EDITOR on the declaration, apply on save" " dark edit apply that file instead (no editor, for scripts)" + " dark edit the whole module at once, same two shapes" + "" + " A MODULE opens every declaration directly under it, and applies them in one" + " batch: all of them are validated first, so a half-finished edit lands nothing" + " rather than half. Leaving a declaration OUT does not end it -- the ones you" + " omitted are named back to you and left alone. `dark remove ` ends a" + " name, explicitly and in one op." "" " The file holds one declaration, the same form `dark module` takes." " `dark view --raw` prints exactly that form, so reading an item," @@ -225,3 +230,201 @@ let complete (args: List) : List = Packages.View.complete state args + + +// --------------------- +// Editing a whole module +// --------------------- +// +// `dark edit ` is the replacement for opening a `.dark` file, and after the flip there is +// no file to open. It renders every declaration directly under the module, hands you $EDITOR, and +// applies the result through `dark module` -- which validates ALL of them before saving ANY, so a +// half-finished edit lands nothing rather than half. +// +// It does NOT delete by omission, and that is deliberate rather than unfinished. `dark module` +// adds and updates; leaving a declaration out of the file does not unbind it. Guessing the other +// way is where this command would lose work: one mis-parse that dropped an item from the render +// would quietly end a name. So omissions are REPORTED and left alone, and `dark remove` stays the +// way a name ends -- one op, confirmed, and visible in `dark status`. + +/// Every declaration directly under , rendered, in a stable order. +/// +/// Types first, then values, then functions, each alphabetical. Order has to be deterministic or +/// a render-edit-save round trip churns the file for nobody's benefit; kind-then-name is the order +/// the module listing already uses, so what you edit looks like what you browsed. +let moduleSource + (branchId: Uuid) + (modulePath: List) + : (String * List) = + let results = Packages.Query.allDirectDescendants branchId modulePath + + // Lambdas here take no type annotation -- Dark does not allow one -- so `wrap` carries the kind + // and the item shape is whatever `LocatedItem` gives: `.location` and `.entity`. + let render (wrap: LanguageTools.ProgramTypes.PackageLocation -> Packages.PackageLocation) (items: List<'a>) : List<(String * String)> = + items + |> Stdlib.List.map (fun i -> i.location) + |> Stdlib.List.sortBy (fun loc -> loc.name) + |> Stdlib.List.filterMap (fun loc -> + match Packages.View.rawSource branchId (wrap loc) with + | Ok src -> Stdlib.Option.Option.Some((loc.name, src)) + | Error _ -> Stdlib.Option.Option.None) + + let rendered = + Stdlib.List.append + (render (fun l -> Packages.PackageLocation.Type l) results.types) + (Stdlib.List.append + (render (fun l -> Packages.PackageLocation.Value l) results.values) + (render (fun l -> Packages.PackageLocation.Function l) results.fns)) + + let names = rendered |> Stdlib.List.map (fun r -> let (n, _) = r in n) + let body = rendered |> Stdlib.List.map (fun r -> let (_, s) = r in s) + + (Stdlib.String.join body "\n\n", names) + + +/// `dark edit `: every declaration under it, in $EDITOR, applied atomically. +let editModule + (state: AppState) + (modulePath: List) + (fileArg: Stdlib.Option.Option) + : AppState = + let branchId = state.currentBranchId + let dotted = Stdlib.String.join modulePath "." + let moduleArg = "/" ++ dotted + + let (before, namesBefore) = moduleSource branchId modulePath + + // Reports which declarations were in the module and are not in , having applied + // nothing about them. `dark module` adds and updates; it does not unbind, and this command + // deliberately does not guess -- one mis-parse that dropped an item from the render would + // otherwise quietly end a name. + // Only after an apply that LANDED. A parse error lands nothing at all, and saying "left alone" + // about it reads as though the rest went in. + let reportOmissions (source: String) : Unit = + let gone = + namesBefore + |> Stdlib.List.filter (fun n -> + Stdlib.Bool.not (Stdlib.String.contains source n)) + + if gone == [] then + () + else + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" left alone, not in your file: {Stdlib.String.join gone ", "}") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + " editing a module does not end a name. `dark remove ` does, explicitly.") + + match fileArg with + | Some file -> + match Stdlib.Cli.File.readText file with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot read {file}: {e.message}") + + state + | Ok source -> + let result = Packages.ModuleCommand.execute state [ moduleArg, file ] + + let _ = + if result.exitCode == 0 then reportOmissions source else () + + result + | None -> + + if namesBefore == [] then + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"{dotted} has no declarations directly under it.") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + " `dark ls` shows what is there; a module of submodules is edited one level down.") + + state + else + + match Stdlib.Cli.Tui.TerminalSupport.current () with + | Unavailable reason -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot open an editor: {reason}.") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" `dark edit {dotted} ` applies a file instead, with no terminal.") + + state + | Available -> + + match Stdlib.Cli.Dir.createTemp () with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {dotted}: no temp directory ({e.message})") + + state + | Ok dir -> + let path = $"{dir}/{Stdlib.String.join modulePath "."}.dark" + + match Stdlib.Cli.File.writeText path before with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {dotted}: could not write {path} ({e.message})") + + state + | Ok _ -> + let editor = editorCommand () + + match Stdlib.Cli.Process.runInteractive editor [ path ] with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Could not run `{editor}`: {e.message}") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" Set $EDITOR, or edit {path} and run `dark edit {dotted} {path}`.") + + state + | Ok code when code != 0 -> + Stdlib.printLine + $"`{editor}` exited {Stdlib.Int.toString code}; nothing applied." + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText $" Your edit is still at {path}.") + + state + | Ok _ -> + match Stdlib.Cli.File.readText path with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Could not read back {path}: {e.message}") + + state + | Ok after -> + if (Stdlib.String.trim after) == (Stdlib.String.trim before) then + let _ = Stdlib.Cli.Dir.deleteRecursive dir + Stdlib.printLine (Stdlib.Cli.UI.Colors.success "no change.") + state + else + let result = Packages.ModuleCommand.execute state [ moduleArg, path ] + + let _ = + if result.exitCode == 0 then reportOmissions after else () + + // Did anything land? Asked against what the module WAS, for the same reason the + // single-item path does: the store pretty-prints, so comparing against what you typed + // says "no" almost always. + // + // The temp file is the only copy of an interactive edit, so it survives a rejection + // and the path is printed. A whole module is a lot to lose. + let (now, _) = moduleSource branchId modulePath + + if (Stdlib.String.trim now) != (Stdlib.String.trim before) then + let _ = Stdlib.Cli.Dir.deleteRecursive dir + result + else + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" nothing landed; your edit is still at {path}") + + result From 7b34464a7c626ab4c6085e4ca99f5d7f2af8af6d Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 17:47:09 -0400 Subject: [PATCH 27/32] Check that the kernel and the package set agree in CI too `refs check` ran as a `scripts/build/compile` action, and CI never runs that script -- it calls `prepare-package-set` and then builds. So the one invariant this whole scheme rests on went unchecked in exactly the place it matters most. It runs wherever the package set is ESTABLISHED now, on both the pinned and the unpinned path, which is the same reason `prepare-package-set` exists at all: one place answers "where does this build's package set come from", and it should answer "and do the two halves agree" in the same breath. --- scripts/build/prepare-package-set | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/build/prepare-package-set b/scripts/build/prepare-package-set index a3a349a56c..612d33db15 100755 --- a/scripts/build/prepare-package-set +++ b/scripts/build/prepare-package-set @@ -38,10 +38,19 @@ import_bundle() { PIN=$(awk '$1 == "commit" { print $2 }' package-set.txt) +# Whether the kernel and the package set agree, asked wherever the package set is established so +# it cannot be forgotten. `scripts/dev/build` asks it too, as a build action; CI never runs that +# script, so without this the one invariant the whole scheme rests on went unchecked in exactly +# the place it matters most. +check_agreement() { + scripts/run-local-exec refs check +} + if [[ -z "$PIN" || "$PIN" == "unset" ]]; then echo "No pin in package-set.txt; building the package set from packages/." scripts/build/reload-packages import_bundle + check_agreement exit 0 fi @@ -54,3 +63,4 @@ scripts/fetch-seed "${URL_ARGS[@]}" --force scripts/run-local-exec refs generate import_bundle +check_agreement From 014de732f72661e812fa79c8562631d760674980 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 17:47:49 -0400 Subject: [PATCH 28/32] Re-pin Cli.executeCliCommand after the edit change A kernel entry point's identity moved, and it is visible here because the file is tracked -- which is the argument Stachu made this morning, paying off the same day. `dark edit ` changed `Packages.Edit.execute`; the registry references it, `Cli.executeCliCommand` references the registry, and a reference is to CONTENT, so the hash walks up. Nothing about the entry point itself changed, which is exactly the kind of move that is invisible without a diff to look at. Reproducible: a fresh reload produces the same file byte for byte. --- backend/src/LibExecution/package-ref-hashes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/LibExecution/package-ref-hashes.txt b/backend/src/LibExecution/package-ref-hashes.txt index 7ce58708a5..0e7838bd0e 100644 --- a/backend/src/LibExecution/package-ref-hashes.txt +++ b/backend/src/LibExecution/package-ref-hashes.txt @@ -1,5 +1,5 @@ fn/Cli.Terminal.renderValue|20a0dff54d9a7e4526e17aa4efe30b98c22df9a897858ddc6e5507bfcfef0724 -fn/Cli.executeCliCommand|a14ca72c5b47dc5bab5adc8777d36b4cb345e14383831e121f34343a10902b46 +fn/Cli.executeCliCommand|a4b840437b7c9b9b426441a3c6efd4a1468af9d822b278ef57befc165f9279a9 fn/Internal.Test.parseSingleTestFromFile|408997505554a133d6f611ba95fcf82fe9f92c6c1b2164ddf1211c6a10caca75 fn/LanguageTools.Parser.CliScript.parseForCli|941945f41a5765708ca908fb58e6129ab5843fbf3159e3b10cce2f703bcdf7a0 fn/LanguageTools.Parser.Parse.parsePTExpr|70bea7c1a45c75f6185e1d3db5677e18b66adf91641cb442ea484e01d3b50ee4 From 2a792918199a0f647a19953b731d39c5fc90639b Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 19:27:40 -0400 Subject: [PATCH 29/32] Move package-ref-hashes.txt with the pin, not with every reload Stachu's call, and it only became available this afternoon. Once type refs resolve from the store BY NAME, the file is a fallback rather than a contract: between pin bumps the committed copy is simply correct, because anything it is stale about resolves by name anyway. So writing it on every reload stopped being necessary -- and writing it on every reload is what made every pair of package-touching branches conflict in a 206-line generated file. That was the cost accepted this morning when the file was first tracked. It no longer has to be paid. A pin bump now produces one reviewable diff naming every kernel identity that moved, which is the signal the file was tracked for in the first place. `generateWith` splits the two halves, because they have different cadences: the hashes must be in MEMORY before values are evaluated, on every reload, or PackageRefs lookups resolve to nothing during it. Only the FILE waits for the pin. Verified by moving a hash and watching nothing happen: changed a package file that shifts `Cli.executeCliCommand`, reloaded, and the file stayed byte-identical while `refs check` still passed 206/206 -- which is the property that makes this safe. --- backend/src/LibDB/PackageRefsGenerator.fs | 25 +++++++++++++++++++++-- backend/src/LocalExec/LocalExec.fs | 8 +++++--- scripts/packages/pin | 13 ++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/backend/src/LibDB/PackageRefsGenerator.fs b/backend/src/LibDB/PackageRefsGenerator.fs index f2c4b383dc..f0e482e29f 100644 --- a/backend/src/LibDB/PackageRefsGenerator.fs +++ b/backend/src/LibDB/PackageRefsGenerator.fs @@ -48,7 +48,19 @@ let private readExistingFile () : Map = /// Query the DB for all current Darklang-owned locations and write /// `package-ref-hashes.txt` in the source tree. -let generate () : Ply = +/// Compute the kernel's ref hashes from the store, set them in memory, and -- when +/// -- write `package-ref-hashes.txt`. +/// +/// The two are separate because they have different cadences. The hashes must be in MEMORY +/// before values are evaluated, on every reload, or `PackageRefs` lookups resolve to nothing +/// during it. The FILE moves only when the pin does. +/// +/// Writing it on every reload is what made every pair of package-touching branches conflict in a +/// 206-line generated file. It stopped being necessary when type refs started resolving from the +/// store by name: the file is a fallback now, not a contract, so between pin bumps the committed +/// copy is simply correct. A pin bump then produces one reviewable diff naming every identity +/// that moved, which is the signal the file was tracked for. +let generateWith (writeToDisk : bool) : Ply = uply { // Collect all referenced items from PackageRefs _lookup maps let typeRefKeys = @@ -165,7 +177,7 @@ let generate () : Ply = // Write the source-tree file (skip if the directory doesn't exist, // e.g. on installed CLIs where the source tree isn't available) let dir = System.IO.Path.GetDirectoryName(sourceTreePath) - if System.IO.Directory.Exists(dir) then + if writeToDisk && System.IO.Directory.Exists(dir) then System.IO.File.WriteAllLines(sourceTreePath, lines |> Array.ofList) let totalWritten = List.length lines print $" Wrote {totalWritten} package ref hashes to {sourceTreePath}" @@ -180,3 +192,12 @@ let generate () : Ply = for key in missing do print $" - {key}" } + + +/// `generateWith`, writing the file. What `refs generate` and `scripts/packages/pin` call. +let generate () : Ply = generateWith true + + +/// `generateWith`, in memory only. What a package RELOAD calls: the hashes have to be current +/// before values are evaluated, and the file is the pin's to move. +let refreshInMemory () : Ply = generateWith false diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index cb7b30af9b..6c1d52a49a 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -58,9 +58,11 @@ module HandleCommand = // work. let! _ = LibDB.Inserts.commitAllAsBaseline "package reload (baseline)" - // Generate hash file BEFORE evaluating values, so that PackageRefs - // lookups resolve correctly during value evaluation. - do! LibDB.PackageRefsGenerator.generate () + // Hashes in MEMORY before evaluating values, so that PackageRefs lookups resolve during + // it. Not written to disk: the file moves when the PIN moves, not on every reload, which + // is what stops two package-touching branches conflicting in it. `scripts/packages/pin` + // and `refs generate` write it. + do! LibDB.PackageRefsGenerator.refreshInMemory () LibExecution.PackageRefs.reloadHashes () // Evaluate all values now that all definitions are in the DB diff --git a/scripts/packages/pin b/scripts/packages/pin index aa5e6b9396..c13660cffe 100755 --- a/scripts/packages/pin +++ b/scripts/packages/pin @@ -86,5 +86,18 @@ OPS=$(field ops) set_field commit "$COMMIT" set_field format "$FORMAT" +# The ref hashes move WITH the pin and only with it. That is what keeps them stable between +# bumps, so ordinary package work never touches the file and two branches that both touch +# packages never conflict in it. The diff this produces is the point: one reviewable list of +# every kernel identity that moved. +# +# Best-effort. The pin is already written, and a store that cannot regenerate (no source tree, +# say) should not lose it. +if scripts/run-local-exec refs generate > /dev/null 2>&1; then + echo "Regenerated package-ref-hashes.txt for the new pin." +else + echo "(could not regenerate package-ref-hashes.txt here; run \`refs generate\` where the store is)" +fi + echo "Pinned to $COMMIT (format $FORMAT, $OPS ops)." echo "Commit package-set.txt: the pin is the record of which package set this kernel was tested against." From fb00fda2fc304ebec828e64ac4c609eb3701f646 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 19:27:40 -0400 Subject: [PATCH 30/32] The server's main takes your namespace, not the reviewed one Stachu's call: locally anyone writes anything; on the shared server you may publish freely to your OWN namespace, and `Darklang.*` goes through a branch and a review. So somebody who has just logged in and written a function can share it without a branch, a PR or a person -- and the namespace the server resolves its own router through still cannot be rebound by a stranger. This is `reservedOwners` reborn at the one edge where it belongs. It used to sit on every write path including local authoring, which was wrong and is why it was deleted this morning. THE THING THAT MADE IT REAL, and it only showed up by running it: the first version refused every push, including the newcomer's own-namespace one. `dark push` sends the whole log, and every client's log carries the `Darklang.*` baseline it was seeded with, so the raw batch always binds reserved names. Only ops the server does NOT already have can change anything -- the rest are content-addressed no-ops -- so those are what gets checked. Without that this rule locks everyone out on their first push. A refusal is a 403 naming the item, not a 500. A client reading "the server broke" has every reason to retry forever. `scmReservedBindings` answers before storing so the handler can say so properly; the rule is still enforced inside `storeOpsWithOwner` as the backstop. Three cases in the `server-folds` gate: your own namespace accepted, a reserved binding refused and named, and the same work on a BRANCH accepted -- because a branch is isolated, nobody runs it, and review is what moves it to main. --- .../Builtins.Matter/Libs/PM/PackageOps.fs | 26 +++ backend/src/LibDB/Inserts.fs | 219 +++++++++++++++--- packages/darklang/sync/relay/protocol.dark | 24 ++ packages/darklang/sync/wire.dark | 12 + scripts/testing/_gates-sync | 44 +++- 5 files changed, 287 insertions(+), 38 deletions(-) diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs index f72a0d8dc7..8bd5285fe5 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs @@ -506,6 +506,32 @@ let fns (pm : PT.PackageManager) : List = // projections. That also meant it could not see what it hosted: `/m` showed "Nothing here" // for packages every client had, a seed could not be cut from the hosted set, and pushing new // code to a server could never change what it ran. All three are wanted, so it folds. + // Asked BEFORE storing, so a refusal can be a refusal rather than a server error. The same + // rule is enforced inside `storeOpsWithOwner` as the backstop -- this exists so the answer can + // carry a status code and a list of names, not so the rule lives in two places. + { name = fn "scmReservedBindings" 0 + typeParams = [] + parameters = + [ Param.make + "records" + (TList(TTuple(TString, TString, [ TString ]))) + "(id, blobHex, originTs) triples" ] + returnType = TList TString + description = + "The reserved names these ops would bind into this store's main, and that it will not accept. Empty means the push is fine." + fn = + (function + | _, _, _, [| DList(_, records) |] -> + uply { + let! names = LibDB.Inserts.reservedBindingsIn (opRecords records) + return Dval.list KTString (names |> List.map Dval.string) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + callEffects = set [ Effect.PackageRead ] + deprecated = NotDeprecated } + { name = fn "scmStoreOps" 0 typeParams = [] parameters = diff --git a/backend/src/LibDB/Inserts.fs b/backend/src/LibDB/Inserts.fs index ac5f611283..da34af6bff 100644 --- a/backend/src/LibDB/Inserts.fs +++ b/backend/src/LibDB/Inserts.fs @@ -560,6 +560,94 @@ let importOpsBulk } +/// The owners a SERVER will not let a push bind into its main. +/// +/// `reservedOwners` used to sit on every write path, including local authoring, and that was +/// wrong -- your own machine is yours, and the check was deleted from all three. This is the same +/// idea at the one edge where it belongs: what a shared server's MAIN will accept from a stranger +/// over HTTP. +/// +/// Why here and not everywhere. Main is what the pin names, what every fetch gets, and what the +/// server resolves its own router through. A push that binds `Darklang.*` there changes what the +/// server serves, for everyone, with no review. Your own namespace is yours to publish to freely: +/// somebody who has just logged in and written a function should be able to share it without a +/// branch, a PR or a person. +/// +/// It stops ACCIDENTS rather than attacks -- the pushed owner is an unsigned string, so this is +/// about what a namespace accepts, not about who is asking. A branch push is unaffected: a branch +/// is isolated, nobody runs it, and review is what moves it to main. +let reservedOnServerMain : Set = Set.ofList [ "Darklang" ] + + +/// The reserved names a batch of ops would bind, if any. Empty means the push is fine. +let private reservedBindings (ops : List) : List = + ops + |> List.choose (fun op -> + let loc = + match op with + | PT.PackageOp.SetName(loc, _, _) -> Some loc + | PT.PackageOp.Decision(_, loc, _, PT.DecisionKind.Override _) -> Some loc + | _ -> None + + match loc with + | Some loc when Set.contains loc.owner reservedOnServerMain -> + let modules = String.concat "." loc.modules + Some( + if modules = "" then + $"{loc.owner}.{loc.name}" + else + $"{loc.owner}.{modules}.{loc.name}" + ) + | _ -> None) + |> List.distinct + + +/// The reserved names these records would bind into this store's main, ignoring ops it already +/// has -- those are content-addressed no-ops whatever they bind. +/// +/// Ignoring them is not an optimisation. `dark push` sends the whole log, and every client's log +/// carries the `Darklang.*` baseline it was seeded with, so checking the raw batch refuses the +/// first push anybody ever makes -- including one that only adds to their own namespace. +let reservedBindingsIn + (records : List) + : Task> = + task { + let valid = + records + |> List.choose (fun (id, blobHex, _) -> + try + Some(System.Guid.Parse id, System.Convert.FromHexString blobHex) + with _ -> + None) + + if List.isEmpty valid then + return [] + else + let! existing = + Sql.query + "SELECT id FROM package_ops WHERE id IN (SELECT value FROM json_each(@ids))" + |> Sql.parameters + [ "ids", + Sql.string ( + "[" + + (valid + |> List.map (fun (id, _) -> "\"" + string id + "\"") + |> String.concat ",") + + "]" + ) ] + |> Sql.executeAsync (fun read -> read.string "id") + + let known = existing |> List.map (fun s -> s.ToLowerInvariant()) |> Set.ofList + + return + valid + |> List.filter (fun (id, _) -> + not (Set.contains ((string id).ToLowerInvariant()) known)) + |> List.choose (fun (id, blob) -> BS.PT.PackageOp.tryDeserialize id blob) + |> reservedBindings + } + + /// RELAY store path: bulk-insert the pushed ops AND record ownership (op_id, owner) in ONE /// transaction. Unlike importOpsBulk this does NOT fold: a relay serves op blobs, not projections. /// The op_owners rows let it serve "your stuff" back by identity. Malformed records are skipped, @@ -590,49 +678,106 @@ let storeOpsWithOwner if List.isEmpty valid then return 0L else - let opRows = - valid - |> List.map (fun (id, blob, ts) -> - [ "id", Sql.uuid id - "op_blob", Sql.bytes blob - "origin_ts", Sql.string ts ]) - - // `effective = 1`: pushed ops ARE folded into this store's main, like any other arriving - // op. The fold happens in the builtin that calls this, and `growIfNeeded` finishes any - // that were interrupted. - // - // This used to be `effective = 0`, so hosted ops were inert data the server served back - // verbatim and never ran. That was a privilege-escalation defence: a name binds - // last-writer-wins across the store, `Darklang.Matter.router` included, so anyone who - // could push could change what the server itself runs. + + // Only ops this store does NOT already have can change anything -- ops are content + // addressed, so a re-push of one already here is a no-op whatever it binds. // - // Given up deliberately, 2026-09-12, with `reservedOwners`. A server that cannot fold - // cannot serve a seed of what it hosts, cannot show it in `/m`, and cannot be deployed to - // by pushing -- and all three are wanted. The replacement is an op's AUTHORITY checked at - // fold time, and it does not exist yet, so for now anyone who can push can rebind - // anything here. - let insertOps = - "INSERT OR IGNORE INTO package_ops (id, op_blob, applied, effective, origin_ts) + // This is not an optimisation, it is the difference between a working rule and one that + // refuses everybody. `dark push` sends the whole log, and every client's log carries the + // `Darklang.*` baseline it was seeded with. Checking the raw batch refuses the first push + // anyone ever makes, including one that only adds to their OWN namespace. Measured: the + // newcomer case failed on `Darklang.Internal.Test.WTTest`, which they had never touched. + let! existing = + Sql.query + "SELECT id FROM package_ops WHERE id IN (SELECT value FROM json_each(@ids))" + |> Sql.parameters + [ "ids", + // Built by hand, as the merge path does: the reflection serializer is disabled + // under AOT. + Sql.string ( + "[" + + (valid + |> List.map (fun (id, _, _) -> "\"" + string id + "\"") + |> String.concat ",") + + "]" + ) ] + |> Sql.executeAsync (fun read -> read.string "id") + + let known = + existing |> List.map (fun s -> s.ToLowerInvariant()) |> Set.ofList + + // Refuse the WHOLE batch, before storing any of it. A push is one act to the person making + // it, and half of it landing is worse than none: the half that lands is live immediately, + // and they have no way to know which half. + let reserved = + valid + |> List.filter (fun (id, _, _) -> + not (Set.contains ((string id).ToLowerInvariant()) known)) + |> List.choose (fun (id, blob, _) -> + BS.PT.PackageOp.tryDeserialize id blob) + |> reservedBindings + + if not (List.isEmpty reserved) then + let shown = reserved |> List.truncate 5 |> String.concat ", " + + let andMore = + if List.length reserved > 5 then + $" (and {List.length reserved - 5} more)" + else + "" + + return + Exception.raiseInternal + ($"this server does not accept pushes that bind {shown}{andMore} into its main. " + + "That namespace is reviewed: push a branch instead (`dark branch push`), and it " + + "lands on main when the change is merged. Your own namespace takes a plain " + + "`dark push`.") + [] + else + + let opRows = + valid + |> List.map (fun (id, blob, ts) -> + [ "id", Sql.uuid id + "op_blob", Sql.bytes blob + "origin_ts", Sql.string ts ]) + + // `effective = 1`: pushed ops ARE folded into this store's main, like any other arriving + // op. The fold happens in the builtin that calls this, and `growIfNeeded` finishes any + // that were interrupted. + // + // This used to be `effective = 0`, so hosted ops were inert data the server served back + // verbatim and never ran. That was a privilege-escalation defence: a name binds + // last-writer-wins across the store, `Darklang.Matter.router` included, so anyone who + // could push could change what the server itself runs. + // + // Given up deliberately, 2026-09-12, with `reservedOwners`. A server that cannot fold + // cannot serve a seed of what it hosts, cannot show it in `/m`, and cannot be deployed to + // by pushing -- and all three are wanted. The replacement is an op's AUTHORITY checked at + // fold time, and it does not exist yet, so for now anyone who can push can rebind + // anything here. + let insertOps = + "INSERT OR IGNORE INTO package_ops (id, op_blob, applied, effective, origin_ts) VALUES (@id, @op_blob, 0, 1, @origin_ts)" - let statements = - if owner = "" then - [ (insertOps, opRows) ] - else - let ownerRows = - valid - |> List.map (fun (id, _, _) -> - [ "op_id", Sql.uuid id; "owner", Sql.string owner ]) + let statements = + if owner = "" then + [ (insertOps, opRows) ] + else + let ownerRows = + valid + |> List.map (fun (id, _, _) -> + [ "op_id", Sql.uuid id; "owner", Sql.string owner ]) - let insertOwners = - "INSERT OR IGNORE INTO op_owners (op_id, owner) VALUES (@op_id, @owner)" + let insertOwners = + "INSERT OR IGNORE INTO op_owners (op_id, owner) VALUES (@op_id, @owner)" - [ (insertOps, opRows); (insertOwners, ownerRows) ] + [ (insertOps, opRows); (insertOwners, ownerRows) ] - // One transaction; the ops-insert counts come first (statement order), so truncate to the - // op rows to report NEW ops rather than owner rows. - let affected = Sql.executeTransactionSync statements - return affected |> List.truncate (List.length opRows) |> List.sumBy int64 + // One transaction; the ops-insert counts come first (statement order), so truncate to the + // op rows to report NEW ops rather than owner rows. + let affected = Sql.executeTransactionSync statements + return affected |> List.truncate (List.length opRows) |> List.sumBy int64 } diff --git a/packages/darklang/sync/relay/protocol.dark b/packages/darklang/sync/relay/protocol.dark index 4e7f3752c1..39fc82ebdf 100644 --- a/packages/darklang/sync/relay/protocol.dark +++ b/packages/darklang/sync/relay/protocol.dark @@ -131,6 +131,30 @@ let pushHandlerFn (req: HttpRequest) : HttpResponse = match SCM.Wire.wireDecode req.body with | Ok bundle -> + // What this server's MAIN will accept from a stranger. Your own namespace takes a plain push; + // the reviewed one does not, and a branch push is how that work arrives instead. 403 rather + // than 500: this is a refusal, and a client that reads it as a server fault retries forever. + let reserved = SCM.Wire.reservedBindingsIn bundle.ops + + if reserved != [] then + let shown = reserved |> Stdlib.List.take 5 |> Stdlib.String.join ", " + + let andMore = + let extra = (Stdlib.List.length reserved) - 5 + + if extra > 0 then + $" (and {Stdlib.Int.toString extra} more)" + else + "" + + Stdlib.Http.responseWithText + ($"this server does not accept pushes that bind {shown}{andMore} into its main. +" + ++ "That namespace is reviewed: `dark branch push` instead, and it lands on main when " + ++ "the change is merged. Your own namespace takes a plain `dark push`.") + 403 + else + match SCM.Wire.storeWithOwner owner bundle.ops bundle.commits with | Ok n -> // A JSON body, not a sentence: the client acts on this count, and a protocol a client diff --git a/packages/darklang/sync/wire.dark b/packages/darklang/sync/wire.dark index eb6caafceb..774e684692 100644 --- a/packages/darklang/sync/wire.dark +++ b/packages/darklang/sync/wire.dark @@ -208,6 +208,18 @@ let localMaxRowid () : Int64 = /// for code every client had: the server's own projection never saw what it hosted. Folding is what /// lets a seed be cut from the hosted set, and what makes pushing new code to a server able to change /// what it runs. +/// The reserved names these ops would bind into this server's main, and that it will not accept. +/// Empty means the push is fine. +/// +/// Asked before storing, so a refusal can answer with a status that says "not allowed" rather than +/// "the server broke" -- a client that sees a 500 has every reason to retry forever. The same rule +/// is enforced inside the store path as the backstop. +let reservedBindingsIn (ops: List) : List = + Builtin.scmReservedBindings ( + ops |> Stdlib.List.map (fun op -> (op.id, op.blobHex, op.ts)) + ) + + let storeWithOwner (owner: String) (ops: List) diff --git a/scripts/testing/_gates-sync b/scripts/testing/_gates-sync index f2365dfa16..68e3d4889b 100644 --- a/scripts/testing/_gates-sync +++ b/scripts/testing/_gates-sync @@ -236,8 +236,50 @@ gate_server_folds() { fail "the startup grow evaluated a pushed val's body" fi + # What a shared server's MAIN accepts from a stranger. Your own namespace takes a plain push; + # the reviewed one does not. This is `reservedOwners` reborn at the one edge where it belongs -- + # the server -- rather than on every local write, where it used to be and was wrong. + cli fn '/Newcomer.MyApp.mine' '() : String = "mine"' > /dev/null 2>&1 + cli commit "my own namespace" -y > /dev/null 2>&1 + out=$(cli push "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"Pushed"* ]]; then + echo " ok a push to your own namespace is accepted" + else + fail "a push to your own namespace was refused: $(head -c 200 <<<"$out")" + fi + + # The whole log goes up, including the `Darklang.*` baseline every client is seeded with, so + # this only works because ops the server already HAS are skipped. Checking the raw batch + # refuses the first push anybody ever makes. + cli fn '/Darklang.Stdlib.List.sneakyGate' '() : Int64 = 1L' > /dev/null 2>&1 + cli commit "reach into the shared namespace" -y > /dev/null 2>&1 + out=$(cli push "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"403"* && "$out" == *"sneakyGate"* ]]; then + echo " ok a push that binds a reserved name is refused, 403, and names it" + else + fail "a reserved-namespace push was not refused as expected: $(head -c 200 <<<"$out")" + fi + + landed=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'sneakyGate';") + [[ "$landed" == "0" ]] || fail "the refused push landed anyway ($landed rows)" + + # A branch carrying the same work is fine: a branch is isolated, nobody runs it, and review is + # what moves it to main. + cli branch gate-shared-fix > /dev/null 2>&1 + cli --branch gate-shared-fix fn '/Darklang.Stdlib.List.viaGateBranch' '() : Int64 = 2L' > /dev/null 2>&1 + out=$(cli branch push gate-shared-fix "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"pushed branch"* ]]; then + echo " ok and the same work on a BRANCH is accepted" + else + fail "a branch push carrying reserved work was refused: $(head -c 200 <<<"$out")" + fi + [[ "$failures" == "0" ]] || exit 1 - echo "server-folds: a push reaches the server's projection, stays out of its draft, and is never run." + echo "server-folds: a push reaches the server's projection, stays out of its draft, is never run, and cannot rebind the reviewed namespace." } gate_relay_routes() { From 41c6150e13dd87bcaf5ca5c003de6c52bf0e2ea5 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 20:10:31 -0400 Subject: [PATCH 31/32] docs: how to work on Dark when the packages live in a store The workflow nobody had written down, and the thing a person actually reaches for: adding a builtin and calling it from Dark, referencing a new package type or function from F#, what your coworker has to do to build your branch, trying the two-machine shape locally, publishing, the pin, and what happens when the op-log format changes. Plus a "things that will bite" section, which is the part I would want if I picked this up cold: `dark log` on a branch shows ops rather than commits, a bundle outliving its branch, a local main drifting ahead of the pin, and two branches adding the same name with different shapes -- a merge conflict git cannot see. Written for both modes. Everything in it works today except where it says otherwise, which matters because the pin ships unset and the reload is still what happens. --- AGENTS.md | 11 +- docs/package-workflow.md | 210 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 docs/package-workflow.md diff --git a/AGENTS.md b/AGENTS.md index 8bffa87f94..ab045bb179 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,9 +71,14 @@ means the first: commit fetched as a seed from a package server, at that commit `scripts/build/prepare-package-set` is the one place that answers that question, and -CI's package-reloading jobs go through it. `scripts/packages/pin` writes the pin; -`dark docs packages` has the rest. The pinned path has never run against a deployed -server, so treat it as written-and-unverified until it has. +CI's package-reloading jobs go through it. `scripts/packages/pin` writes the pin. The +pinned path has never run against a deployed server, so treat it as +written-and-unverified until it has. + +**`docs/package-workflow.md` is the how-to**: adding a builtin and calling it from +Dark, referencing a new package type or fn from F#, what your coworker does to build +your branch, publishing, the pin, format changes, and what will bite. `dark docs +packages` is the short version from inside the CLI. The container builds once when it starts. Rebuild-on-save is available but off by default, because a five-file change under a watcher pays for five rebuilds, four of them diff --git a/docs/package-workflow.md b/docs/package-workflow.md new file mode 100644 index 0000000000..ef03dffcf6 --- /dev/null +++ b/docs/package-workflow.md @@ -0,0 +1,210 @@ +# Working on Dark when the packages live in a store + +How to add a builtin, use it from Dark, reference a new package type or function from F#, try all +of it locally, and get it to everybody else. + +The one-line version: **the store is the source of truth for Dark code, git is the source of truth +for F#, and a git branch carries the package work its F# depends on so the two merge as one thing.** + +Read `docs/dev-setup` first if the container is not up. `dark docs packages` is the short version of +this from inside the CLI. + +--- + +## Where a build's packages come from + +Two answers, and `package-set.txt` at the root says which: + + commit unset built from `packages/` on disk, by reloading it + commit fetched as a SEED from a package server, at that commit + +`scripts/build/prepare-package-set` is the one place that answers the question, and CI's +package-reloading jobs go through it. It also checks that the kernel and the package set agree +before letting the build continue. + +It ships `unset` today, which is the reloading behaviour that has always been there. Everything +below works in both modes except where it says otherwise. + +--- + +## Adding a builtin and using it from Dark + +A builtin is F#; the Dark that calls it is a package item. They are two sides of one change and +they belong in one PR. + +1. Add the fn to the `fns` list in the right `Builtins/Libs/.fs`. `AGENTS.md` says how to + pick the subproject and why you return structured Dark values rather than strings. +2. Wrap it, once, in a Dark package fn. `tests/builtin` enforces exactly one wrapper and at least + one caller: a builtin with two `Builtin.x` references fails, and so does one with none. +3. `scripts/dev/build`. The build now ends with: + + All 206 kernel refs resolve, and all 734 builtins this package set calls + exist in this kernel. + + That second half is the check that matters here. It reads `package_builtin_deps`, which the fold + fills from the real call graph -- not a grep over text, which is what it replaces and which stops + being possible once `packages/` is gone. + +**Removing or renaming a builtin is two steps, and the check enforces the order.** Land the package +change that stops calling it, move the pin forward, and only then remove the builtin. Doing it the +other way round fails the build with: + + this package set calls 1 builtin(s) this kernel does not have: + Builtin.someBuiltinWeDeleted (v0) + +Adding one is safe in a single step, because an older package set simply does not call it. + +--- + +## Referencing a new package type or function from F# + +This is the case that needs a branch, and the only one that does. + +F# names package items through `PackageRefs`. Those resolve **from the store, by name**, with the +hash in `package-ref-hashes.txt` as a fallback and a shape check in between: a candidate whose +signature (fn) or declaration (type) does not match what this build was compiled against is refused, +loudly, and the pinned version is used. A type nobody has pinned yet -- one your branch has just +authored -- has nothing to compare against, so the store's answer is taken. That is what makes this +work at all. + + git checkout -b add-foo + dark branch add-foo # same name; they travel together + + dark type /Darklang.LanguageTools.Foo '{ n: Int64 }' + # authored on the dark branch. Invisible from main, and from everyone else. + + # ...add `let foo = p [] "Foo"` in PackageRefs.fs and use it... + + scripts/dev/build # compiles; regenerates refs; runs `refs check` + scripts/packages/bundle export # ~1KB of JSON: package-branch.json + git commit -a # F# + package-ref-hashes.txt + the bundle + +The bundle is how your package work reaches anyone else. Export is explicit, like `git add`. + +**Order matters in one small way:** author the type before the F# that uses it RUNS. Adding the ref +and building is fine -- a ref is a lazy closure -- but the first code path that reaches an +unresolvable one raises. + +### What your coworker does + +Nothing special. They check out the branch and build. + + git checkout add-foo + scripts/dev/build # prepare-package-set imports the bundle automatically + +Importing is automatic because checking out the git branch IS asking for that branch's package +code, and it is idempotent -- ops are content addressed, so a second import says "up to date". + +**If they are on the wrong dark branch, the build tells them, completely, at build time:** + + 1 kernel ref(s) do not resolve against this package set: + type Darklang.LanguageTools.Foo + + git is on `add-foo` and there is a dark branch called `add-foo`, + but you are on dark main. Try `dark switch add-foo`. + +That is deliberate: the refs are lazy, so without this you would find out whenever some unrelated +command happened to reach that code path. + +### Reviewing it + + scripts/packages/bundle show --source + +Prints what the bundle changes, with the full declarations, by importing into a throwaway store. +Reviewing somebody's branch should not mean taking their ops into the store you work in. + +--- + +## Trying it locally before it goes anywhere + +Everything above is local already. Your store is yours: there are no restrictions on what you can +author or rebind on your own machine, including under `Darklang.*`. + +To try the two-machine shape without a server, a second store is enough: + + dark branch export add-foo /tmp/b.json # from one + dark branch import /tmp/b.json # into the other + +To try it against a real server, run one: + + DARK_MATTER_WRITE_SECRET= dark serve Darklang.Matter.router --port 9090 + +`scripts/testing/gates seed-serving` and `gates server-folds` do exactly this and are the worked +examples if you want to see the whole thing driven end to end. + +--- + +## Getting it to everybody else + + dark push your own namespace, straight to the server's main + dark branch push anything else, including `Darklang.*` + +**Your namespace is yours to publish to freely.** Log in, write a function, push it. No branch, no +PR, no waiting. + +**`Darklang.*` is reviewed**, and the server refuses a direct push that binds it: + + 403: this server does not accept pushes that bind Darklang.Stdlib.List.foo + into its main. That namespace is reviewed: `dark branch push` instead, and + it lands on main when the change is merged. + +A branch push is always accepted, because a branch is isolated, nobody runs it, and review is what +moves it to main. That is also why an abandoned branch costs nothing: close the PR and the bundle +goes with it. + +The server FOLDS what you push, so it shows up in `/m`, `/p` and in seeds. It never RUNS it: a +pushed `val` is folded, browsable and servable, and evaluated only on the machine that fetches it. + +--- + +## The pin, and when it moves + +`package-set.txt` names the commit a build's packages come from. It moves when somebody re-pins: + + scripts/packages/pin head --url # pin to what it has now + scripts/packages/pin --show + scripts/packages/pin --unset # back to building from packages/ + +Every pin is checked before it is written, so a commit that does not resolve is refused then rather +than at the next CI run. + +Re-pinning also regenerates `package-ref-hashes.txt`, and that is the only thing that does. +Ordinary package work leaves it alone, which is what stops two branches that both touch packages +conflicting in a 206-line generated file. The diff a pin bump produces is the point: one reviewable +list of every kernel identity that moved. + +A pinned seed is cached per machine under `~/.darklang/seeds`, so the network is needed once per +pin, not once per clone. + +--- + +## When the op-log format changes + +A store does not get rebuilt from text any more, so it has to be carried forward in place. + + dark store format, which commit it was cut at, which build cut it + dark store upgrade move the op log to this build's format + dark store rollback put back the copy `upgrade` took + +`upgrade` copies the store first and does the rewrite in one transaction. It only rewrites BLOBS: a +change that moves content hashes is a much larger migration and it refuses rather than half-doing +it. It is a no-op until the first real format bump. + +If you are running a build that reads an OLDER format than your store, it says so and names the file +to move back -- a `mv` needs no working binary, and by then you do not have one. + +--- + +## Things that will bite + +- **`dark log` on a branch shows that branch's OPS, not main's commits.** Ask `dark --branch main + log` when you want commits. +- **A bundle left behind after its branch merges** is how somebody imports work from weeks ago. + Going back to dark main removes it; `bundle export` on main will clean up a stale one. +- **Your main can drift ahead of the pin** if you `dark pull`. Then your branch may rest on commits + nobody else has, and your bundle will not be enough for them. `bundle export` warns when it sees + this. +- **Two branches adding the same name with different shapes** both merge, last-writer-wins picks + one, and the loser's F# then references a name whose declaration is not what it compiled against. + The compatibility check catches it at the second merge, so main goes red rather than silently + wrong -- but it is a merge conflict git cannot see. From 773480a17b8709cd33a113aeeee26c9997531dc1 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Sat, 12 Sep 2026 23:12:45 -0400 Subject: [PATCH 32/32] Gotchas: the traps the worklist was holding The test lock and why a broad pkill is the wrong way to clear it, Sqlite's @p0 parameters, the registry sweep taxing the suite, the Dark syntax traps, and measuring the artifact people actually run. These were in a working document that has now been folded away; they belong where agents read them. --- AGENTS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index ab045bb179..543751f3c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -339,6 +339,26 @@ op log directly. ## Gotchas +**The test lock.** `run-backend-tests` refuses if another run holds `rundir/test.lock`. Wait for +it. Do not clear it with a broad `pkill -f "out/Tests"`: that pattern matches every sibling clone +on this machine and will kill somebody else's suite. Scope it to the clone if you must +(`pkill -f "boot-migrate/backend/Build/out/Tests"`). + +**`Stdlib.Sqlite` parameters are `@p0`, `@p1`, not `?`.** With `?` nothing matches and nothing +errors, so a cache silently never fills. + +**A new CLI command joins the registry sweep the day it is registered**, and the sweep runs every +command with a bogus argument. An expensive command therefore taxes the whole suite; `grep` cost +nine minutes until it learned to refuse an unscoped search. + +**Dark syntax traps.** No `let private`. No `rec` keyword. The list separator is `,`. A comment +inside a list literal breaks the parser. Parenthesise a piped qualified call: +`(Mod.f x) |> ...`. + +**Measure the artifact people actually run.** Debug, `publish -c Release`, R2R and AOT differ by +about 25x on startup. Three separate wrong conclusions in one week came from measuring the wrong +one. + **PackageRefs stale hash.** `backend/src/LibExecution/package-ref-hashes.txt` isn't in git. Empty is tolerated; non-empty with a missing key crashes at startup with "PackageRefs: X hash not found". After adding a ref: